vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wip: generate NDF table

Browse source
This commit is contained in:
jaime-m-p 2024-05-07 20:39:22 +02:00
parent a5fa2fec60
commit def3d13a9d
1 changed files with 12 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

23
scripts/gen-unicode-data.py
View file

@ -100,14 +100,15 @@ print_cat("map", "lowercase", map_lowercase)
print_cat("map", "uppercase", map_uppercase) print_cat("map", "uppercase", map_uppercase)
inv_map_nfd = {} # TODO: this is wrong
for codepoint in range(0x110000): # inv_map_nfd = {}
char = chr(codepoint) # for codepoint in range(0x110000):
norm = ord(unicodedata.normalize('NFD', char)[0]) # char = chr(codepoint)
if codepoint != norm: # norm = ord(unicodedata.normalize('NFD', char)[0])
a, b = inv_map_nfd.get(norm, (codepoint, codepoint)) # if codepoint != norm:
inv_map_nfd[norm] = (min(a, codepoint), max(b, codepoint)) # a, b = inv_map_nfd.get(norm, (codepoint, codepoint))
nfd_ranges = [ (a, b, nfd) for nfd,(a,b) in inv_map_nfd.items() ] # inv_map_nfd[norm] = (min(a, codepoint), max(b, codepoint))
nfd_ranges = list(sorted(nfd_ranges)) # nfd_ranges = [ (a, b, nfd) for nfd,(a,b) in inv_map_nfd.items() ]
del inv_map_nfd # nfd_ranges = list(sorted(nfd_ranges))
print_cat("range_value", "nfd", nfd_ranges) # del inv_map_nfd
# print_cat("range_value", "nfd", nfd_ranges)