Multi-line quoted strings support.

* grub-core/script/lexer.c (append_newline): Removed.
	(grub_script_lexer_yywrap): Refactored.
	(grub_script_lexer_init): Refactored.
	* grub-core/script/yylex.l (yywrap): New function.
	(grub_lexer_resplit): New function.
	(grub_lexer_unput): New function.
	* include/grub/script_sh.h (grub_lexer_param): New members, unput
	and resplit.
	* tests/grub_script_echo1.in: Added few more testcases.

ChangeLog

@@ -1,3 +1,17 @@
+2010-09-04  BVK Chaitanya  <bvk.groups@gmail.com>
+
+	Multi-line quoted strings support.
+
+	* grub-core/script/lexer.c (append_newline): Removed.
+	(grub_script_lexer_yywrap): Refactored.
+	(grub_script_lexer_init): Refactored.
+	* grub-core/script/yylex.l (yywrap): New function.
+	(grub_lexer_resplit): New function.
+	(grub_lexer_unput): New function.
+	* include/grub/script_sh.h (grub_lexer_param): New members, unput
+	and resplit.
+	* tests/grub_script_echo1.in: Added few more testcases.
+
 2010-09-04  Vladimir Serbinenko  <phcoder@gmail.com>
 
 	* grub-core/kern/misc.c: Don't add abort alias in utils.

grub-core/script/lexer.c

@@ -98,8 +98,6 @@ grub_script_lexer_record_stop (struct grub_parser_param *parser)
   return result;
 }
 
-#define MAX(a,b) ((a) < (b) ? (b) : (a))
-
 /* Record STR if input recording is enabled.  */
 void
 grub_script_lexer_record (struct grub_parser_param *parser, char *str)
@@ -115,7 +113,7 @@ grub_script_lexer_record (struct grub_parser_param *parser, char *str)
   if (lexer->recordpos + len + 1 > lexer->recordlen)
     {
       old = lexer->recording;
-      lexer->recordlen = MAX (len, lexer->recordlen) * 2;
+      lexer->recordlen = grub_max (len, lexer->recordlen) * 2;
       lexer->recording = grub_realloc (lexer->recording, lexer->recordlen);
       if (!lexer->recording)
 	{
@@ -131,84 +129,93 @@ grub_script_lexer_record (struct grub_parser_param *parser, char *str)
   lexer->recordpos += len;
 }
 
-/* Append '\n' to SRC, before '\0'  */
-static char *
-append_newline (const char *src)
-{
-  char *line;
-  grub_size_t len;
-
-  len = grub_strlen (src);
-  line = grub_malloc (len + 2);
-  if (!line)
-    return 0;
-
-  grub_strcpy (line, src);
-
-  line[len] = '\n';
-  line[len + 1] = '\0';
-  return line;
-}
-
 /* Read next line of input if necessary, and set yyscanner buffers.  */
 int
-grub_script_lexer_yywrap (struct grub_parser_param *parserstate)
+grub_script_lexer_yywrap (struct grub_parser_param *parserstate,
+			  const char *input)
 {
   int len;
-  char *line;
+  char *p = 0;
-  char *line2;
+  char *line = 0;
   YY_BUFFER_STATE buffer;
   struct grub_lexer_param *lexerstate = parserstate->lexerstate;
 
-  if (!lexerstate->refs)
+  if (! lexerstate->refs && ! lexerstate->prefix && ! input)
-    return 0;
+    return 1;
 
-  if (!lexerstate->getline)
+  if (! lexerstate->getline && ! input)
     {
       grub_script_yyerror (parserstate, "unexpected end of file");
-      return 0;
+      return 1;
     }
 
   line = 0;
-  buffer = 0;
+  if (! input)
     lexerstate->getline (&line, 1);
+  else
+    line = grub_strdup (input);
+
+  /* Ensure '\n' at the end.  */
+  if (line && line[0] == '\0')
+    {
+      grub_free (line);
+      line = grub_strdup ("\n");
+    }
+
+  if (line && (len = grub_strlen(line)) && line[len - 1] != '\n')
+    {
+      p = grub_realloc (line, len + 2);
+      if (p)
+	{
+	  p[len++] = '\n';
+	  p[len] = '\0';
+	}
+      line = p;
+    }
+
   if (! line)
     {
-      grub_script_yyerror (parserstate, 0); /* XXX this could be for ^C case? */
+      grub_script_yyerror (parserstate, "out of memory");
-      return 0;
+      return 1;
     }
 
-  len = grub_strlen (line);
+  /* Prepend any left over unput-text.  */
-  if (line[len - 1] == '\n')
+  if (lexerstate->prefix)
     {
-      buffer = yy_scan_string (line, lexerstate->yyscanner);
+      int plen = grub_strlen (lexerstate->prefix);
-    }
-  else
-    {
-      line2 = append_newline (line);
-      if (line2)
-	{
-	  buffer = yy_scan_string (line2, lexerstate->yyscanner);
-	  grub_free (line2);
-	}
-    }
 
+      p = grub_malloc (len + plen + 1);
+      if (! p)
+	{
 	  grub_free (line);
+	  return 1;
+	}
+      grub_strcpy (p, lexerstate->prefix);
+      lexerstate->prefix = 0;
+
+      grub_strcpy (p + plen, line);
+      grub_free (line);
+
+      line = p;
+      len = len + plen;
+    }
+
+  buffer = yy_scan_string (line, lexerstate->yyscanner);
+  grub_free (line);
+
   if (! buffer)
     {
       grub_script_yyerror (parserstate, 0);
-      return 0;
-    }
-
       return 1;
     }
+  return 0;
+}
 
 struct grub_lexer_param *
 grub_script_lexer_init (struct grub_parser_param *parser, char *script,
 			grub_reader_getline_t getline)
 {
   int len;
-  char *script2;
   YY_BUFFER_STATE buffer;
   struct grub_lexer_param *lexerstate;
 
@@ -232,34 +239,18 @@ grub_script_lexer_init (struct grub_parser_param *parser, char *script,
       return 0;
     }
 
-  buffer = 0;
+  yyset_extra (parser, lexerstate->yyscanner);
-  script = script ? : "\n";
+  parser->lexerstate = lexerstate;
-  len = grub_strlen (script);
 
-  if (len != 0 && script[len - 1] == '\n')
+  if (grub_script_lexer_yywrap (parser, script ?: "\n"))
-    {
-      buffer = yy_scan_string (script, lexerstate->yyscanner);
-    }
-  else
-    {
-      script2 = append_newline (script);
-      if (script2)
-	{
-	  buffer = yy_scan_string (script2, lexerstate->yyscanner);
-	  grub_free (script2);
-	}
-    }
-
-  if (!buffer)
     {
+      parser->lexerstate = 0;
       yylex_destroy (lexerstate->yyscanner);
       grub_free (lexerstate->yyscanner);
-
       grub_free (lexerstate->text);
       grub_free (lexerstate);
       return 0;
     }
-  yyset_extra (parser, lexerstate->yyscanner);
 
   return lexerstate;
 }

grub-core/script/yylex.l

@@ -58,6 +58,9 @@
 #define YY_INPUT(buf,res,max) do { res = 0; } while (0)
 
 /* forward declarations */
+static int grub_lexer_unput (const char *input, yyscan_t yyscanner);
+static int grub_lexer_resplit (const char *input, yyscan_t yyscanner);
+
 static void  grub_lexer_yyfree (void *, yyscan_t yyscanner);
 static void* grub_lexer_yyalloc (yy_size_t, yyscan_t yyscanner);
 static void* grub_lexer_yyrealloc (void*, yy_size_t, yyscan_t yyscanner);
@@ -99,7 +102,7 @@ typedef size_t yy_size_t;
 %option never-interactive
 
 %option noyyfree noyyalloc noyyrealloc
-%option nounistd nostdinit nodefault noyylineno noyywrap
+%option nounistd nostdinit nodefault noyylineno
 
 /* Reduce lexer size, by not defining these.  */
 %option noyy_top_state
@@ -118,13 +121,17 @@ CHAR            [^{}|&$;<> \t\n\'\"\\]
 DIGITS          [[:digit:]]+
 NAME            [[:alpha:]_][[:alnum:]_]*
 
-ESC             \\.
+ESC             \\(.|\n)
+SQCHR           [^\']
+DQCHR           {ESC}|[^\\\"]
+DQSTR           \"{DQCHR}*\"
+SQSTR           \'{SQCHR}*\'
 SPECIAL         \?|\#|\*|\@
 VARIABLE        ${NAME}|$\{{NAME}\}|${DIGITS}|$\{{DIGITS}\}|${SPECIAL}|$\{{SPECIAL}\}
-DQSTR           \"([^\\\"]|{ESC})*\"
-SQSTR           \'[^\']*\'
 WORD            ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
 
+MULTILINE       {WORD}?((\"{DQCHR}*)|(\'{SQCHR}*)|(\\\n))
+
 %x              SPLIT
 %x              DQUOTE
 %x              SQUOTE
@@ -171,27 +178,24 @@ WORD            ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
 "function"      { RECORD; return GRUB_PARSER_TOKEN_FUNCTION;  }
 "menuentry"     { RECORD; return GRUB_PARSER_TOKEN_MENUENTRY; }
 
+{MULTILINE}     {
+                  if (grub_lexer_unput (yytext, yyscanner))
+		    return GRUB_PARSER_TOKEN_BAD;
+		}
+
 {NAME}          { RECORD; return GRUB_PARSER_TOKEN_NAME; }
 {WORD}          {
                   RECORD;
-                  /* resplit yytext */
-		  grub_dprintf ("lexer", "word: [%s]\n", yytext);
 		  yypush_buffer_state (YY_CURRENT_BUFFER, yyscanner);
-                  if (yy_scan_string (yytext, yyscanner))
+		  if (grub_lexer_resplit (yytext, yyscanner))
 		    {
-                      yyextra->lexerstate->merge_start = 1;
-                      yy_push_state (SPLIT, yyscanner);
-                    }
-                  else
-                    {
-                      grub_script_yyerror (yyextra, 0);
 		      yypop_buffer_state (yyscanner);
 		      return GRUB_PARSER_TOKEN_WORD;
 		    }
+		  yyextra->lexerstate->resplit = 1;
                 }
-
+.               {
-.|\n            {
+                  grub_script_yyerror (yyextra, yytext);
-                  grub_script_yyerror (yyextra, "unrecognized token");
 		  return GRUB_PARSER_TOKEN_BAD;
                 }
 
@@ -199,6 +203,7 @@ WORD            ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
 
 <SPLIT>{
   \\.           { COPY (yytext + 1, yyleng - 1); }
+  \\\n          { /* ignore */ }
   \"            {
                   yy_push_state (DQUOTE, yyscanner);
                   ARG (GRUB_SCRIPT_ARG_TYPE_TEXT);
@@ -216,6 +221,7 @@ WORD            ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
   <<EOF>>       {
                   yy_pop_state (yyscanner);
                   yypop_buffer_state (yyscanner);
+		  yyextra->lexerstate->resplit = 0;
                   yyextra->lexerstate->merge_end = 1;
                   ARG (GRUB_SCRIPT_ARG_TYPE_TEXT);
                 }
@@ -273,15 +279,20 @@ WORD            ({CHAR}|{DQSTR}|{SQSTR}|{ESC}|{VARIABLE})+
 
 <<EOF>>         {
                   yypop_buffer_state (yyscanner);
-                  if (! grub_script_lexer_yywrap (yyextra))
-                    {
 		  yyextra->lexerstate->eof = 1;
 		  return GRUB_PARSER_TOKEN_EOF;
                 }
-                }
-
 %%
 
+int
+yywrap (yyscan_t yyscanner)
+{
+  if (yyget_extra (yyscanner)->lexerstate->resplit)
+    return 1;
+
+  return grub_script_lexer_yywrap (yyget_extra (yyscanner), 0);
+}
+
 static void
 grub_lexer_yyfree (void *ptr, yyscan_t yyscanner __attribute__ ((unused)))
 {
@@ -301,8 +312,6 @@ grub_lexer_yyrealloc (void *ptr, yy_size_t size,
   return grub_realloc (ptr, size);
 }
 
-#define MAX(a,b) ((a) < (b) ? (b) : (a))
-
 static void copy_string (struct grub_parser_param *parser, const char *str, unsigned hint)
 {
   int size;
@@ -312,7 +321,7 @@ static void copy_string (struct grub_parser_param *parser, const char *str, unsi
   len = hint ? hint : grub_strlen (str);
   if (parser->lexerstate->used + len >= parser->lexerstate->size)
     {
-      size = MAX (len, parser->lexerstate->size) * 2;
+      size = grub_max (len, parser->lexerstate->size) * 2;
       ptr = grub_realloc (parser->lexerstate->text, size);
       if (!ptr)
         {
@@ -326,3 +335,34 @@ static void copy_string (struct grub_parser_param *parser, const char *str, unsi
   grub_strcpy (parser->lexerstate->text + parser->lexerstate->used - 1, str);
   parser->lexerstate->used += len;
 }
+
+static int
+grub_lexer_resplit (const char *text, yyscan_t yyscanner)
+{
+  /* resplit text */
+  if (yy_scan_string (text, yyscanner))
+    {
+      yyget_extra (yyscanner)->lexerstate->merge_start = 1;
+      yy_push_state (SPLIT, yyscanner);
+      return 0;
+    }
+  grub_script_yyerror (yyget_extra (yyscanner), 0);
+  return 1;
+}
+
+static int
+grub_lexer_unput (const char *text, yyscan_t yyscanner)
+{
+  struct grub_lexer_param *lexerstate = yyget_extra (yyscanner)->lexerstate;
+
+  if (lexerstate->prefix)
+    grub_free (lexerstate->prefix);
+
+  lexerstate->prefix = grub_strdup (text);
+  if (! lexerstate->prefix)
+    {
+      grub_script_yyerror (yyget_extra (yyscanner), "out of memory");
+      return 1;
+    }
+  return 0;
+}

include/grub/script_sh.h

@@ -194,6 +194,12 @@ struct grub_lexer_param
   /* Type of text.  */
   grub_script_arg_type_t type;
 
+  /* Flag to indicate resplit in progres.  */
+  unsigned resplit;
+
+  /* Text that is unput.  */
+  char *prefix;
+
   /* Flex scanner.  */
   void *yyscanner;
 
@@ -289,7 +295,7 @@ void grub_script_lexer_ref (struct grub_lexer_param *);
 void grub_script_lexer_deref (struct grub_lexer_param *);
 void grub_script_lexer_record_start (struct grub_parser_param *);
 char *grub_script_lexer_record_stop (struct grub_parser_param *);
-int  grub_script_lexer_yywrap (struct grub_parser_param *);
+int  grub_script_lexer_yywrap (struct grub_parser_param *, const char *input);
 void grub_script_lexer_record (struct grub_parser_param *, char *);
 
 /* Functions to track allocated memory.  */

tests/grub_script_echo1.in

@@ -57,3 +57,45 @@ e"$foo"${bar}o hello world
 
 foo=echo
 $foo 1234
+
+echo "one
+"
+echo "one
+\""
+echo "one
+two"
+
+echo one"two
+"three
+echo one"two
+\""three
+echo one"two
+\"three\"
+four"
+
+
+echo 'one
+'
+echo 'one
+\'
+echo 'one
+two'
+echo one'two
+'
+echo one'two
+\'
+echo one'two
+\'three
+
+# echo "one\
+# two"
+# echo 'one\
+# two'
+# echo foo\
+# bar
+# \
+# echo foo
+# echo "one
+#
+# two"
+