Group combining diacritics

autogen.sh

@@ -13,7 +13,7 @@ echo timestamp > stamp-h.in
 
 python util/import_gcry.py lib/libgcrypt/ .
 
-python util/import_bidi.py util/UnicodeData.txt unidata.c
+python util/import_unicode.py util/UnicodeData.txt unidata.c
 
 for rmk in conf/*.rmk ${GRUB_CONTRIB}/*/conf/*.rmk; do
   if test -e $rmk ; then

font/font.c

@@ -27,7 +27,7 @@
 #include <grub/video.h>
 #include <grub/bitmap.h>
 #include <grub/charset.h>
-#include <grub/bidi.h>
+#include <grub/unicode.h>
 
 #ifndef FONT_DEBUG
 #define FONT_DEBUG 0
@@ -882,7 +882,10 @@ grub_font_get_string_width (grub_font_t font, const char *str)
        grub_utf8_to_ucs4 (&code, 1, ptr, -1, &ptr) > 0; )
     {
       glyph = grub_font_get_glyph_with_fallback (font, code);
+      if (glyph)
 	width += glyph->device_width;
+      else
+	width += unknown_glyph->device_width;
     }
 
   return width;
@@ -989,13 +992,210 @@ grub_font_get_glyph_with_fallback (grub_font_t font, grub_uint32_t code)
         }
     }
 
-  if (best_glyph)
   return best_glyph;
-  else
-    /* Glyph not available in any font.  Return unknown glyph.  */
-    return unknown_glyph;
 }
 
+static struct grub_font_glyph *
+grub_font_dup_glyph (struct grub_font_glyph *glyph)
+{
+  static struct grub_font_glyph *ret;
+  ret = grub_malloc (sizeof (*ret) + (glyph->width * glyph->height + 7) / 8);
+  if (!ret)
+    return NULL;
+  grub_memcpy (ret, glyph, sizeof (*ret)
+	       + (glyph->width * glyph->height + 7) / 8);
+  return ret;
+}
+
+/* FIXME: suboptimal.  */
+static void
+grub_font_blit_glyph (struct grub_font_glyph *target,
+		      struct grub_font_glyph *src,
+		      unsigned dx, unsigned dy)
+{
+  unsigned src_bit, tgt_bit, src_byte, tgt_byte;
+  unsigned i, j;
+  for (i = 0; i < src->height; i++)
+    {
+      src_bit = (src->width * i) % 8;
+      src_byte = (src->width * i) / 8;
+      tgt_bit = (target->width * (dy + i) + dx) % 8;
+      tgt_byte = (target->width * (dy + i) + dx) / 8;
+      for (j = 0; j < src->width; j++)
+	{
+	  target->bitmap[tgt_byte] |= ((src->bitmap[src_byte] << src_bit)
+				       & 0x80) >> tgt_bit;
+	  src_bit++;
+	  tgt_bit++;
+	  if (src_bit == 8)
+	    {
+	      src_byte++;
+	      src_bit = 0;
+	    }
+	  if (tgt_bit == 8)
+	    {
+	      tgt_byte++;
+	      tgt_bit = 0;
+	    }
+	}
+    }
+}
+
+static inline enum grub_comb_type
+get_comb_type (grub_uint32_t c)
+{
+  static grub_uint8_t *comb_types = NULL;
+  struct grub_unicode_compact_range *cur;
+
+  if (!comb_types)
+    {
+      unsigned i;
+      comb_types = grub_zalloc (GRUB_UNICODE_MAX_CACHED_CHAR);
+      if (comb_types)
+	for (cur = grub_unicode_compact; cur->end; cur++)
+	  for (i = cur->start; i <= cur->end
+		 && i < GRUB_UNICODE_MAX_CACHED_CHAR; i++)
+	    comb_types[i] = cur->comb_type;
+      else
+	grub_errno = GRUB_ERR_NONE;
+    }
+
+  if (comb_types && c < GRUB_UNICODE_MAX_CACHED_CHAR)
+    return comb_types[c];
+
+  for (cur = grub_unicode_compact; cur->end; cur++)
+    if (cur->start <= c && c <= cur->end)
+      return cur->comb_type;
+
+  return GRUB_BIDI_TYPE_L;
+}
+
+static struct grub_font_glyph *
+grub_font_construct_glyph (grub_font_t hinted_font,
+			   const struct grub_unicode_glyph *glyph_id)
+{
+  grub_font_t font;
+  grub_uint16_t width;
+  grub_uint16_t height;
+  grub_int16_t offset_x;
+  grub_int16_t offset_y;
+  grub_uint16_t device_width;
+  struct grub_font_glyph *main_glyph;
+  struct grub_font_glyph **combining_glyphs;
+  unsigned i;
+  struct grub_font_glyph *glyph;
+
+  main_glyph = grub_font_get_glyph_with_fallback (hinted_font, glyph_id->base);
+
+  if (!main_glyph)
+    {
+      /* Glyph not available in any font.  Return unknown glyph.  */
+      return grub_font_dup_glyph (unknown_glyph);
+    }
+
+  if (!glyph_id->ncomb)
+    return grub_font_dup_glyph (main_glyph);
+
+  combining_glyphs = grub_malloc (sizeof (combining_glyphs[0])
+				  * glyph_id->ncomb);
+  if (!combining_glyphs)
+    {
+      grub_errno = GRUB_ERR_NONE;
+      return grub_font_dup_glyph (main_glyph);
+    }
+  
+  font = main_glyph->font;
+  width = main_glyph->width;
+  height = main_glyph->height;
+  offset_x = main_glyph->offset_x;
+  offset_y = main_glyph->offset_y;
+  device_width = main_glyph->device_width;
+
+  for (i = 0; i < glyph_id->ncomb; i++)
+    {
+      enum grub_comb_type combtype;
+      combining_glyphs[i]
+	= grub_font_get_glyph_with_fallback (font, glyph_id->combining[i]);
+      if (!combining_glyphs[i])
+	continue;
+      combtype = get_comb_type (glyph_id->combining[i]);
+      switch (combtype)
+	{
+	default:
+	  {
+	    /* Default handling. Just draw combining character on top
+	       of base character.
+	       FIXME: support more unicode types correctly.
+	    */
+	    grub_int16_t nx = main_glyph->device_width
+	      + combining_glyphs[i]->offset_x;
+
+	    device_width += combining_glyphs[i]->device_width;
+	    if (nx < offset_x)
+	      {
+		width += offset_x - nx;
+		offset_x = nx;
+	      }
+	    if (offset_y > combining_glyphs[i]->offset_y)
+	      {
+		height += offset_y - combining_glyphs[i]->offset_y;
+		offset_y = combining_glyphs[i]->offset_y;
+	      }
+	    if (nx + combining_glyphs[i]->width - offset_x >= width)
+	      width = nx + combining_glyphs[i]->width - offset_x + 1;
+	    if (height
+		< (combining_glyphs[i]->height 
+		   + combining_glyphs[i]->offset_y) - offset_y)
+	      height = (combining_glyphs[i]->height 
+			+ combining_glyphs[i]->offset_y) - offset_y;
+	  }
+	}
+    }
+
+  glyph = grub_zalloc (sizeof (*glyph) + (width * height + 7) / 8);
+  if (!glyph)
+    {
+      grub_errno = GRUB_ERR_NONE;
+      return grub_font_dup_glyph (main_glyph);
+    }
+
+  glyph->font = font;
+  glyph->width = width;
+  glyph->height = height;
+  glyph->offset_x = offset_x;
+  glyph->offset_y = offset_y;
+  glyph->device_width = device_width;
+
+  grub_font_blit_glyph (glyph, main_glyph, main_glyph->offset_x - offset_x,
+			(height + offset_y)
+			- (main_glyph->height + main_glyph->offset_y));
+
+  for (i = 0; i < glyph_id->ncomb; i++)
+    {
+      enum grub_comb_type combtype;
+      if (!combining_glyphs[i])
+	continue;
+      combtype = get_comb_type (glyph_id->combining[i]);
+      switch (combtype)
+	{
+	default:
+	  {
+	    /* Default handling. Just draw combining character on top
+	       of base character.
+	       FIXME: support more unicode types correctly.
+	    */
+	    grub_font_blit_glyph (glyph, combining_glyphs[i],
+				  main_glyph->device_width
+				  + combining_glyphs[i]->offset_x - offset_x,
+				  (height + offset_y)
+				  - (combining_glyphs[i]->height 
+				     + combining_glyphs[i]->offset_y));
+	  }
+	}
+    }
+
+  return glyph;
+}
 
 /* Draw the specified glyph at (x, y).  The y coordinate designates the
    baseline of the character, while the x coordinate designates the left
@@ -1051,27 +1251,27 @@ static inline enum grub_bidi_type
 get_bidi_type (grub_uint32_t c)
 {
   static grub_uint8_t *bidi_types = NULL;
-  struct grub_bidi_compact_range *cur;
+  struct grub_unicode_compact_range *cur;
 
   if (!bidi_types)
     {
       unsigned i;
-      bidi_types = grub_zalloc (GRUB_BIDI_MAX_CACHED_UNICODE_CHAR);
+      bidi_types = grub_zalloc (GRUB_UNICODE_MAX_CACHED_CHAR);
       if (bidi_types)
-	for (cur = grub_bidi_compact; cur->end; cur++)
+	for (cur = grub_unicode_compact; cur->end; cur++)
 	  for (i = cur->start; i <= cur->end
-		 && i < GRUB_BIDI_MAX_CACHED_UNICODE_CHAR; i++)
+		 && i < GRUB_UNICODE_MAX_CACHED_CHAR; i++)
-	    bidi_types[i] = cur->type;
+	    bidi_types[i] = cur->bidi_type;
       else
 	grub_errno = GRUB_ERR_NONE;
     }
 
-  if (bidi_types && c < GRUB_BIDI_MAX_CACHED_UNICODE_CHAR)
+  if (bidi_types && c < GRUB_UNICODE_MAX_CACHED_CHAR)
     return bidi_types[c];
 
-  for (cur = grub_bidi_compact; cur->end; cur++)
+  for (cur = grub_unicode_compact; cur->end; cur++)
     if (cur->start <= c && c <= cur->end)
-      return cur->type;
+      return cur->bidi_type;
 
   return GRUB_BIDI_TYPE_L;
 }
@@ -1079,7 +1279,7 @@ get_bidi_type (grub_uint32_t c)
 static grub_ssize_t
 grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
 				 grub_size_t logical_len,
-				 grub_uint32_t **visual)
+				 struct grub_unicode_glyph **visual_out)
 {
   enum grub_bidi_type type = GRUB_BIDI_TYPE_L;
   enum override_status {OVERRIDE_NEUTRAL = 0, OVERRIDE_R, OVERRIDE_L};
@@ -1092,9 +1292,9 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
   enum override_status stack_override[GRUB_BIDI_MAX_EXPLICIT_LEVEL + 3];
   unsigned stack_depth = 0;
   unsigned invalid_pushes = 0;
-  unsigned no_markers_len = 0;
+  unsigned visual_len = 0;
   unsigned run_start, run_end;
-  grub_uint32_t *no_markers;
+  struct grub_unicode_glyph *visual;
   unsigned cur_level;
 
   auto void push_stack (unsigned new_override, unsigned new_level);
@@ -1130,13 +1330,13 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
   auto void revert (unsigned start, unsigned end);
   void revert (unsigned start, unsigned end)
   {
-    grub_uint32_t t;
+    struct grub_unicode_glyph t;
     unsigned k, tl;
     for (k = 0; k <= (end - start) / 2; k++)
       {
-	t = no_markers[start+k];
+	t = visual[start+k];
-	no_markers[start+k] = no_markers[end-k];
+	visual[start+k] = visual[end-k];
-	no_markers[end-k] = t;
+	visual[end-k] = t;
 	tl = levels[start+k];
 	levels[start+k] = levels[end-k];
 	levels[end-k] = tl;
@@ -1154,8 +1354,8 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
       return -1;
     }
 
-  no_markers = grub_malloc (sizeof (resolved_types[0]) * logical_len);
+  visual = grub_malloc (sizeof (visual[0]) * logical_len);
-  if (!no_markers)
+  if (!visual)
     {
       grub_free (resolved_types);
       grub_free (levels);
@@ -1178,6 +1378,27 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
   cur_override = OVERRIDE_NEUTRAL;
   for (i = 0; i < logical_len; i++)
     {
+      /* Variation selectors >= 17 are outside of BMP and SMP. 
+	 Handle variation selectors first to avoid potentially costly lookups.
+       */
+      if (logical[i] >= GRUB_UNICODE_VARIATION_SELECTOR_1
+	  && logical[i] <= GRUB_UNICODE_VARIATION_SELECTOR_16)
+	{
+	  if (!visual_len)
+	    continue;
+	  visual[visual_len - 1].variant
+	    = logical[i] - GRUB_UNICODE_VARIATION_SELECTOR_1 + 1;
+	}
+      if (logical[i] >= GRUB_UNICODE_VARIATION_SELECTOR_17
+	  && logical[i] <= GRUB_UNICODE_VARIATION_SELECTOR_256)
+	{
+	  if (!visual_len)
+	    continue;
+	  visual[visual_len - 1].variant
+	    = logical[i] - GRUB_UNICODE_VARIATION_SELECTOR_17 + 17;
+	  continue;
+	}
+
       type = get_bidi_type (logical[i]);
       switch (type)
 	{
@@ -1199,28 +1420,54 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
 	case GRUB_BIDI_TYPE_BN:
 	  break;
 	default:
-	  levels[no_markers_len] = cur_level;
+	  {
+	    enum grub_comb_type comb_type;
+	    comb_type = get_comb_type (logical[i]);
+	    if (comb_type)
+	      {
+		grub_uint32_t *n;
+		if (!visual_len)
+		  break;
+		n = grub_realloc (visual[visual_len - 1].combining, 
+				  sizeof (grub_uint32_t)
+				  * (visual[visual_len - 1].ncomb + 1));
+		if (!n)
+		  {
+		    grub_errno = GRUB_ERR_NONE;
+		    break;
+		  }
+		visual[visual_len - 1].combining = n;
+		visual[visual_len - 1].combining[visual[visual_len - 1].ncomb]
+		  = logical[i];
+		visual[visual_len - 1].ncomb++;
+		break;
+	      }
+	    levels[visual_len] = cur_level;
 	    if (cur_override != OVERRIDE_NEUTRAL)
-	    resolved_types[no_markers_len] = 
+	      resolved_types[visual_len] = 
 		(cur_override == OVERRIDE_L) ? GRUB_BIDI_TYPE_L : GRUB_BIDI_TYPE_R;
 	    else
-	    resolved_types[no_markers_len] = type;
+	      resolved_types[visual_len] = type;
-	  no_markers[no_markers_len] = logical[i];
+	    visual[visual_len].base = logical[i];
-	  no_markers_len++;
+	    visual[visual_len].variant = 0;
+	    visual[visual_len].ncomb = 0;
+	    visual[visual_len].combining = NULL;
+	    visual_len++;
+	  }
 	}
     }
 
-  for (run_start = 0; run_start < no_markers_len; run_start = run_end)
+  for (run_start = 0; run_start < visual_len; run_start = run_end)
     {
       unsigned prev_level, next_level, cur_run_level;
       unsigned last_type, last_strong_type;
-      for (run_end = run_start; run_end < no_markers_len &&
+      for (run_end = run_start; run_end < visual_len &&
 	     levels[run_end] == levels[run_start]; run_end++);
       if (run_start == 0)
 	prev_level = base_level;
       else
 	prev_level = levels[run_start - 1];
-      if (run_end == no_markers_len)
+      if (run_end == visual_len)
 	next_level = base_level;
       else
 	next_level = levels[run_end];
@@ -1350,7 +1597,7 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
 	}
     }
 
-  for (i = 0; i < no_markers_len; i++)
+  for (i = 0; i < visual_len; i++)
     {
       if (!(levels[i] & 1) && resolved_types[i] == GRUB_BIDI_TYPE_R)
 	{
@@ -1377,7 +1624,7 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
     unsigned min_odd_level = 0xffffffff;
     unsigned max_level = 0;
     unsigned j;
-    for (i = 0; i < no_markers_len; i++)
+    for (i = 0; i < visual_len; i++)
       {
 	if (levels[i] > max_level)
 	  max_level = levels[i];
@@ -1387,11 +1634,11 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
     for (j = max_level; j >= min_odd_level; j--)
       {
 	unsigned in = 0;
-	for (i = 0; i < no_markers_len; i++)
+	for (i = 0; i < visual_len; i++)
 	  {
 	    if (i != 0 && levels[i] >= j && levels[i-1] < j)
 	      in = i;
-	    if (levels[i] >= j && (i + 1 == no_markers_len || levels[i+1] < j))
+	    if (levels[i] >= j && (i + 1 == visual_len || levels[i+1] < j))
 	      revert (in, i);
 	  }
       }
@@ -1399,8 +1646,8 @@ grub_err_bidi_logical_to_visual (grub_uint32_t *logical,
 
   grub_free (levels);
 
-  *visual = no_markers;
+  *visual_out = visual;
-  return no_markers_len;
+  return visual_len;
 }
 
 /* Draw a UTF-8 string of text on the current video render target.
@@ -1415,8 +1662,9 @@ grub_font_draw_string (const char *str, grub_font_t font,
 {
   int x;
   struct grub_font_glyph *glyph;
-  grub_uint32_t *logical, *visual, *ptr;
+  grub_uint32_t *logical;
   grub_ssize_t logical_len, visual_len;
+  struct grub_unicode_glyph *visual, *ptr;
 
   logical_len = grub_utf8_to_ucs4_alloc (str, &logical, 0);
   if (logical_len < 0)
@@ -1429,10 +1677,14 @@ grub_font_draw_string (const char *str, grub_font_t font,
 
   for (ptr = visual, x = left_x; ptr < visual + visual_len; ptr++)
     {
-      glyph = grub_font_get_glyph_with_fallback (font, *ptr);
+      grub_err_t err;
-      if (grub_font_draw_glyph (glyph, color, x, baseline_y)
+      glyph = grub_font_construct_glyph (font, ptr);
-          != GRUB_ERR_NONE)
+      if (!glyph)
 	return grub_errno;
+      err = grub_font_draw_glyph (glyph, color, x, baseline_y);
+      grub_free (glyph);
+      if (err)
+	return err;
       x += glyph->device_width;
     }
 

include/grub/unicode.h

@@ -21,11 +21,12 @@
 
 #include <grub/types.h>
 
-struct grub_bidi_compact_range
+struct grub_unicode_compact_range
 {
-  grub_uint32_t start:24;
+  grub_uint32_t start:21;
-  grub_uint32_t end:24;
+  grub_uint32_t end:21;
-  grub_uint8_t type;
+  grub_uint8_t bidi_type:5;
+  grub_uint8_t comb_type;
 } __attribute__ ((packed));
 
 enum grub_bidi_type
@@ -51,9 +52,36 @@ enum grub_bidi_type
     GRUB_BIDI_TYPE_ON
   };
 
-extern struct grub_bidi_compact_range grub_bidi_compact[];
+enum grub_comb_type
+  {
+    GRUB_UNICODE_STACK_BELOW = 220,
+    GRUB_UNICODE_STACK_ABOVE = 230,
+    /* If combining nature is indicated only by class and
+       not "combining type".  */
+    GRUB_UNICODE_COMB_ME = 253,
+    GRUB_UNICODE_COMB_MC = 254,
+    GRUB_UNICODE_COMB_MN = 255,
+  };
 
-#define GRUB_BIDI_MAX_CACHED_UNICODE_CHAR 0x20000
+/* This structure describes a glyph as opposed to character.  */
+struct grub_unicode_glyph
+{
+  grub_uint32_t base;
+  /* Unicode permits 256 variation selectors but since we need a value
+     for "default" we have to use grub_uint16_t.  */
+  grub_uint16_t variant;
+  grub_size_t ncomb;
+  grub_uint32_t *combining;
+};
+
+#define GRUB_UNICODE_VARIATION_SELECTOR_1 0xfe00
+#define GRUB_UNICODE_VARIATION_SELECTOR_16 0xfe0f
+#define GRUB_UNICODE_VARIATION_SELECTOR_17 0xe0100
+#define GRUB_UNICODE_VARIATION_SELECTOR_256 0xe01ef
+
+extern struct grub_unicode_compact_range grub_unicode_compact[];
+
+#define GRUB_UNICODE_MAX_CACHED_CHAR 0x20000
 /*  Unicode mandates an arbitrary limit.  */
 #define GRUB_BIDI_MAX_EXPLICIT_LEVEL 61
 

util/grub-mkfont.c

@@ -170,6 +170,9 @@ add_char (struct grub_font_info *font_info, FT_Face face,
   if ((*p_glyph) && ((*p_glyph)->char_code == char_code))
     return;
 
+  if (glyph->next)
+    printf ("%x\n", char_code);
+
   width = glyph->bitmap.width;
   height = glyph->bitmap.rows;
 

util/import_bidi.py

@@ -1,52 +0,0 @@
-#*
-#*  GRUB  --  GRand Unified Bootloader
-#*  Copyright (C) 2010  Free Software Foundation, Inc.
-#*
-#*  GRUB is free software: you can redistribute it and/or modify
-#*  it under the terms of the GNU General Public License as published by
-#*  the Free Software Foundation, either version 3 of the License, or
-#*  (at your option) any later version.
-#*
-#*  GRUB is distributed in the hope that it will be useful,
-#*  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#*  GNU General Public License for more details.
-#*
-#*  You should have received a copy of the GNU General Public License
-#*  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
-#*
-
-import re
-import sys
-import os
-import datetime
-
-if len (sys.argv) < 3:
-    print ("Usage: %s SOURCE DESTINATION" % sys.argv[0])
-    exit (0)
-infile = open (sys.argv[1], "r")
-outfile = open (sys.argv[2], "w")
-outfile.write ("#include <grub/bidi.h>\n")
-outfile.write ("\n")
-outfile.write ("struct grub_bidi_compact_range grub_bidi_compact[] = {\n")
-
-begin = -2
-last = -2
-lasttype = "X"
-for line in infile:
-    sp = line.split (";")
-    cur = int (sp[0], 16)
-    curtype = sp[4]
-    if last + 1 != cur or curtype != lasttype:
-        if begin != -2 and lasttype != "L":
-            outfile.write (("{0x%x, 0x%x, GRUB_BIDI_TYPE_%s},\n" \
-                                % (begin, last, lasttype)))
-        begin = cur
-    last = cur
-    lasttype = curtype
-if lasttype != "L":
-    outfile.write (("{0x%x, 0x%x, GRUB_BIDI_TYPE_%s},\n" \
-                        % (begin, last, lasttype)))
-outfile.write ("{0, 0, 0},\n")
-
-outfile.write ("};")

util/import_unicode.py

@@ -0,0 +1,67 @@
+#*
+#*  GRUB  --  GRand Unified Bootloader
+#*  Copyright (C) 2010  Free Software Foundation, Inc.
+#*
+#*  GRUB is free software: you can redistribute it and/or modify
+#*  it under the terms of the GNU General Public License as published by
+#*  the Free Software Foundation, either version 3 of the License, or
+#*  (at your option) any later version.
+#*
+#*  GRUB is distributed in the hope that it will be useful,
+#*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#*  GNU General Public License for more details.
+#*
+#*  You should have received a copy of the GNU General Public License
+#*  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
+#*
+
+import re
+import sys
+import os
+import datetime
+
+if len (sys.argv) < 3:
+    print ("Usage: %s SOURCE DESTINATION" % sys.argv[0])
+    exit (0)
+infile = open (sys.argv[1], "r")
+outfile = open (sys.argv[2], "w")
+outfile.write ("#include <grub/unicode.h>\n")
+outfile.write ("\n")
+outfile.write ("struct grub_unicode_compact_range grub_unicode_compact[] = {\n")
+
+begincode = -2
+lastcode = -2
+lastbiditype = "X"
+lastcombtype = -1
+for line in infile:
+    sp = line.split (";")
+    curcode = int (sp[0], 16)
+    curbiditype = sp[4]
+    curcombtype = int (sp[3], 10)
+    if curcombtype <= 255 and curcombtype >= 253:
+        print ("UnicodeData.txt uses combination type %d. Conflict." \
+                   % curcombtype)
+        raise
+    if curcombtype == 0 and sp[2] == "Me":
+        curcombtype = 253
+    if curcombtype == 0 and sp[2] == "Mc":
+        curcombtype = 254
+    if curcombtype == 0 and sp[2] == "Mn":
+        curcombtype = 255
+    if lastcode + 1 != curcode or curbiditype != lastbiditype \
+            or curcombtype != lastcombtype:
+        if begincode != -2 and (lastbiditype != "L" or lastcombtype != 0):
+            outfile.write (("{0x%x, 0x%x, GRUB_BIDI_TYPE_%s, %d},\n" \
+                                % (begincode, lastcode, lastbiditype, \
+                                       lastcombtype)))
+        begincode = curcode
+    lastcode = curcode
+    lastbiditype = curbiditype
+    lastcombtype = curcombtype
+if lastbiditype != "L" or lastcombtype != 0:
+    outfile.write (("{0x%x, 0x%x, GRUB_BIDI_TYPE_%s, %d},\n" \
+                        % (begincode, lastcode, lastbiditype, lastcombtype)))
+outfile.write ("{0, 0, 0, 0},\n")
+
+outfile.write ("};")