removed arbitrary limit on token size

This commit is contained in:
BVK Chaitanya 2010-01-23 00:14:00 +05:30
parent 547e494f1b
commit df6dc2113a
3 changed files with 89 additions and 81 deletions

View file

@ -157,8 +157,10 @@ struct grub_lexer_param
int merge_start; int merge_start;
int merge_end; int merge_end;
/* Text of current token. */ /* Part of a multi-part token. */
char *text; char *text;
unsigned used;
unsigned size;
/* Type of text. */ /* Type of text. */
grub_script_arg_type_t type; grub_script_arg_type_t type;
@ -168,12 +170,9 @@ struct grub_lexer_param
/* Flex scanner buffer. */ /* Flex scanner buffer. */
void *buffer; void *buffer;
/* Length of current token text. */
unsigned size;
}; };
#define GRUB_LEXER_TOKEN_MAX 256 #define GRUB_LEXER_INITIAL_TEXT_SIZE 32
#define GRUB_LEXER_RECORD_INCREMENT 256 #define GRUB_LEXER_RECORD_INCREMENT 256
/* State of the parser as passes to the parser. */ /* State of the parser as passes to the parser. */

View file

@ -213,7 +213,8 @@ grub_script_lexer_init (struct grub_parser_param *parser, char *script,
if (!lexerstate) if (!lexerstate)
return 0; return 0;
lexerstate->text = grub_malloc (GRUB_LEXER_TOKEN_MAX); lexerstate->size = GRUB_LEXER_INITIAL_TEXT_SIZE;
lexerstate->text = grub_malloc (lexerstate->size);
if (!lexerstate->text) if (!lexerstate->text)
{ {
grub_free (lexerstate); grub_free (lexerstate);
@ -301,7 +302,7 @@ grub_script_yylex (union YYSTYPE *value,
do do
{ {
/* Empty lexerstate->text. */ /* Empty lexerstate->text. */
lexerstate->size = 0; lexerstate->used = 1;
lexerstate->text[0] = '\0'; lexerstate->text[0] = '\0';
token = yylex (value, lexerstate->yyscanner); token = yylex (value, lexerstate->yyscanner);
@ -311,7 +312,6 @@ grub_script_yylex (union YYSTYPE *value,
/* Merging feature uses lexerstate->text instead of yytext. */ /* Merging feature uses lexerstate->text instead of yytext. */
if (lexerstate->merge_start) if (lexerstate->merge_start)
{ {
lexerstate->text[lexerstate->size] = '\0';
str = lexerstate->text; str = lexerstate->text;
type = lexerstate->type; type = lexerstate->type;
} }

View file

@ -37,31 +37,19 @@
grub_printf ("fatal error: %s\n", msg); \ grub_printf ("fatal error: %s\n", msg); \
} while (0) } while (0)
#define PUSH(c) \ #define COPY(str, hint) \
do { \ do { \
if (yyextra->lexerstate->size >= GRUB_LEXER_TOKEN_MAX - 1) \ copy_string (yyextra, str, hint); \
grub_script_yyerror (yyextra, "token too long"); \
else \
yyextra->lexerstate->text[yyextra->lexerstate->size++] = c; \
} while (0)
#define COPY(str) \
do { \
char *ptr = str; \
while (*ptr && ! yyextra->err) \
{ \
PUSH (*ptr); \
ptr++; \
} \
} while (0) } while (0)
#define RECORD \
do { \ #define RECORD \
grub_script_lexer_record (yyextra, yytext); \ do { \
grub_script_lexer_record (yyextra, yytext); \
} while (0) } while (0)
#define ARG(t) \ #define ARG(t) \
do { \ do { \
yyextra->lexerstate->type = t; \ yyextra->lexerstate->type = t; \
return GRUB_PARSER_TOKEN_WORD; \ return GRUB_PARSER_TOKEN_WORD; \
} while (0) } while (0)
@ -73,6 +61,8 @@
static void grub_lexer_yyfree (void *, yyscan_t yyscanner); static void grub_lexer_yyfree (void *, yyscan_t yyscanner);
static void* grub_lexer_yyalloc (yy_size_t, yyscan_t yyscanner); static void* grub_lexer_yyalloc (yy_size_t, yyscan_t yyscanner);
static void* grub_lexer_yyrealloc (void*, yy_size_t, yyscan_t yyscanner); static void* grub_lexer_yyrealloc (void*, yy_size_t, yyscan_t yyscanner);
static void copy_string (struct grub_parser_param *, const char *,
unsigned hint);
%} %}
@ -201,52 +191,47 @@ WORD ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
RECORD; RECORD;
/* resplit yytext */ /* resplit yytext */
yypush_buffer_state (YY_CURRENT_BUFFER, yyscanner); yypush_buffer_state (YY_CURRENT_BUFFER, yyscanner);
if (yy_scan_string (yytext, yyscanner)) if (yy_scan_string (yytext, yyscanner))
{ {
yyextra->lexerstate->merge_start = 1; yyextra->lexerstate->merge_start = 1;
yy_push_state (SPLIT, yyscanner); yy_push_state (SPLIT, yyscanner);
} }
else else
{ {
grub_script_yyerror (yyextra, 0); grub_script_yyerror (yyextra, 0);
yypop_buffer_state (yyscanner); yypop_buffer_state (yyscanner);
return GRUB_PARSER_TOKEN_WORD; return GRUB_PARSER_TOKEN_WORD;
} }
} }
.|\n { .|\n {
grub_script_yyerror (yyextra, "unrecognized token"); grub_script_yyerror (yyextra, "unrecognized token");
return GRUB_PARSER_TOKEN_BAD; return GRUB_PARSER_TOKEN_BAD;
} }
/* Split word into multiple args */ /* Split word into multiple args */
<SPLIT>{ <SPLIT>{
\\. { PUSH (yytext[1]); } \\. { COPY (yytext + 1, yyleng - 1); }
\" { \" {
yy_push_state (DQUOTE, yyscanner); yy_push_state (DQUOTE, yyscanner);
ARG (GRUB_SCRIPT_ARG_TYPE_TEXT); ARG (GRUB_SCRIPT_ARG_TYPE_TEXT);
} }
\' { \' {
yy_push_state (SQUOTE, yyscanner); yy_push_state (SQUOTE, yyscanner);
ARG (GRUB_SCRIPT_ARG_TYPE_TEXT); ARG (GRUB_SCRIPT_ARG_TYPE_TEXT);
} }
\$ { \$ {
yy_push_state (VAR, yyscanner); yy_push_state (VAR, yyscanner);
ARG (GRUB_SCRIPT_ARG_TYPE_TEXT); ARG (GRUB_SCRIPT_ARG_TYPE_TEXT);
} }
{CHAR} { PUSH (yytext[0]); } \\ |
.|\n { [^\"\'$\\]+ { COPY (yytext, yyleng); }
/* This cannot happen. */
grub_script_yyerror (yyextra, "internal error: unexpected characters in a word");
return GRUB_PARSER_TOKEN_BAD;
}
<<EOF>> { <<EOF>> {
yy_pop_state (yyscanner); yy_pop_state (yyscanner);
yypop_buffer_state (yyscanner); yypop_buffer_state (yyscanner);
yyextra->lexerstate->merge_end = 1; yyextra->lexerstate->merge_end = 1;
ARG (GRUB_SCRIPT_ARG_TYPE_TEXT); ARG (GRUB_SCRIPT_ARG_TYPE_TEXT);
} }
} }
@ -255,23 +240,23 @@ WORD ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
\? | \? |
{DIGITS} | {DIGITS} |
{NAME} { {NAME} {
COPY (yytext); COPY (yytext, yyleng);
yy_pop_state (yyscanner); yy_pop_state (yyscanner);
if (YY_START == SPLIT) if (YY_START == SPLIT)
ARG (GRUB_SCRIPT_ARG_TYPE_VAR); ARG (GRUB_SCRIPT_ARG_TYPE_VAR);
else else
ARG (GRUB_SCRIPT_ARG_TYPE_DQVAR); ARG (GRUB_SCRIPT_ARG_TYPE_DQVAR);
} }
\{\?\} | \{\?\} |
\{{DIGITS}\} | \{{DIGITS}\} |
\{{NAME}\} { \{{NAME}\} {
yytext[yyleng - 1] = '\0'; yytext[yyleng - 1] = '\0';
COPY (yytext + 1); COPY (yytext + 1, yyleng - 2);
yy_pop_state (yyscanner); yy_pop_state (yyscanner);
if (YY_START == SPLIT) if (YY_START == SPLIT)
ARG (GRUB_SCRIPT_ARG_TYPE_VAR); ARG (GRUB_SCRIPT_ARG_TYPE_VAR);
else else
ARG (GRUB_SCRIPT_ARG_TYPE_DQVAR); ARG (GRUB_SCRIPT_ARG_TYPE_DQVAR);
} }
.|\n { return GRUB_PARSER_TOKEN_BAD; } .|\n { return GRUB_PARSER_TOKEN_BAD; }
} }
@ -279,33 +264,34 @@ WORD ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
<SQUOTE>{ <SQUOTE>{
\' { \' {
yy_pop_state (yyscanner); yy_pop_state (yyscanner);
ARG (GRUB_SCRIPT_ARG_TYPE_SQSTR); ARG (GRUB_SCRIPT_ARG_TYPE_SQSTR);
} }
(.|\n) { PUSH (yytext[0]); } [^\']+ { COPY (yytext, yyleng); }
} }
<DQUOTE>{ <DQUOTE>{
\" { \" {
yy_pop_state (yyscanner); yy_pop_state (yyscanner);
ARG (GRUB_SCRIPT_ARG_TYPE_DQSTR); ARG (GRUB_SCRIPT_ARG_TYPE_DQSTR);
} }
\$ { \$ {
yy_push_state (VAR, yyscanner); yy_push_state (VAR, yyscanner);
ARG (GRUB_SCRIPT_ARG_TYPE_DQSTR); ARG (GRUB_SCRIPT_ARG_TYPE_DQSTR);
} }
\\\\ { PUSH ('\\'); } \\\\ { COPY ("\\", 1); }
\\\" { PUSH ('\"'); } \\\" { COPY ("\"", 1); }
\\\n { /* ignore */ } \\\n { /* ignore */ }
(.|\n) { PUSH (yytext[0]); } [^\"$\\\n]+ { COPY (yytext, yyleng); }
(.|\n) { COPY (yytext, yyleng); }
} }
<<EOF>> { <<EOF>> {
yypop_buffer_state (yyscanner); yypop_buffer_state (yyscanner);
if (! grub_script_lexer_yywrap (yyextra)) if (! grub_script_lexer_yywrap (yyextra))
{ {
yyextra->lexerstate->eof = 1; yyextra->lexerstate->eof = 1;
return GRUB_PARSER_TOKEN_EOF; return GRUB_PARSER_TOKEN_EOF;
} }
} }
%% %%
@ -329,3 +315,26 @@ grub_lexer_yyrealloc (void *ptr, yy_size_t size,
return grub_realloc (ptr, size); return grub_realloc (ptr, size);
} }
static void copy_string (struct grub_parser_param *parser, const char *str, unsigned hint)
{
int len;
int size;
char *ptr;
len = hint ? hint : grub_strlen (str);
if (parser->lexerstate->used + len >= parser->lexerstate->size)
{
size = parser->lexerstate->size * 2;
ptr = grub_realloc (parser->lexerstate->text, size);
if (!ptr)
{
grub_script_yyerror (parser, 0);
return;
}
parser->lexerstate->text = ptr;
parser->lexerstate->size = size;
}
grub_strcpy (parser->lexerstate->text + parser->lexerstate->used - 1, str);
parser->lexerstate->used += len;
}