diff options
author | Steven G. Johnson <stevenj@mit.edu> | 2019-03-30 16:05:50 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-03-30 16:05:50 -0400 |
commit | fd4d8a3454c7c2feffa1cb10a4c518ebb338e48c (patch) | |
tree | 850dff8b48017b3cd68c3e0c5ae34cad704e1d55 /data | |
parent | 94395db2821f15a97adaa18ea4dd10f84023af44 (diff) | |
download | libutf8proc-fd4d8a3454c7c2feffa1cb10a4c518ebb338e48c.tar.gz libutf8proc-fd4d8a3454c7c2feffa1cb10a4c518ebb338e48c.tar.bz2 |
give up on Unifont for charwidth data (#150)
* fix CHARBOUND option for non-characters
* give up on unifont for charwidth computation
Diffstat (limited to 'data')
-rw-r--r-- | data/Makefile | 19 | ||||
-rw-r--r-- | data/charwidths.jl | 56 |
2 files changed, 9 insertions, 66 deletions
diff --git a/data/Makefile b/data/Makefile index 4ef0ced..c824a9b 100644 --- a/data/Makefile +++ b/data/Makefile @@ -1,7 +1,7 @@ # Unicode data generation rules. Except for the test data files, most # users will not use these Makefile rules, which are primarily to re-generate # unicode_data.c when we get a new Unicode version or charwidth data; they -# require ruby, fontforge, and julia to be installed. +# require ruby and julia to be installed. # programs CURL=curl @@ -9,7 +9,6 @@ RUBY=ruby PERL=perl MAKE=make JULIA=julia -FONTFORGE=fontforge CURLFLAGS = --retry 5 --location .PHONY: clean @@ -19,19 +18,7 @@ CURLFLAGS = --retry 5 --location utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt emoji-data.txt $(RUBY) data_generator.rb < UnicodeData.txt > $@ -# GNU Unifont version for font metric calculations: -UNIFONT_VERSION=12.0.01 - -unifont.ttf: - $(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf - -unifont_upper.ttf: - $(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf - -%.sfd: %.ttf - $(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);" - -CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt +CharWidths.txt: charwidths.jl EastAsianWidth.txt $(JULIA) charwidths.jl > $@ # Unicode data version @@ -65,5 +52,5 @@ emoji-data.txt: $(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://unicode.org/Public/emoji/`echo $(UNICODE_VERSION) | cut -d. -f1-2`/emoji-data.txt clean: - rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd emoji-data.txt + rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt emoji-data.txt rm -f utf8proc_data.c.new diff --git a/data/charwidths.jl b/data/charwidths.jl index 2d4c033..9b3d508 100644 --- a/data/charwidths.jl +++ b/data/charwidths.jl @@ -1,9 +1,12 @@ # Following work by @jiahao, we compute character widths using a combination of -# * advance widths from GNU Unifont (advance width 512 = 1 en) +# * character category # * UAX 11: East Asian Width # * a few exceptions as needed # Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734 # +# We used to also use data from GNU Unifont, but that has proven unreliable +# and unlikely to match widths assumed by terminals. +# # Requires Julia (obviously) and FontForge. ############################################################################# @@ -49,7 +52,7 @@ const UTF8PROC_CATEGORY_CO = 29 ############################################################################# # Use a default width of 1 for all character categories that are # letter/symbol/number-like, as well as for unassigned/private-use chars. -# This can be overriden by Unifont or UAX 11 +# This can be overriden by UAX 11 # below, but provides a useful nonzero fallback for new codepoints when # a new Unicode version has been released but Unifont hasn't been updated yet. @@ -58,7 +61,6 @@ push!(zerowidth, UTF8PROC_CATEGORY_MN) push!(zerowidth, UTF8PROC_CATEGORY_MC) push!(zerowidth, UTF8PROC_CATEGORY_ME) push!(zerowidth, UTF8PROC_CATEGORY_SK) -push!(zerowidth, UTF8PROC_CATEGORY_ZS) push!(zerowidth, UTF8PROC_CATEGORY_ZL) push!(zerowidth, UTF8PROC_CATEGORY_ZP) push!(zerowidth, UTF8PROC_CATEGORY_CC) @@ -71,44 +73,8 @@ for c in 0x0000:0x110000 end ############################################################################# -# Widths from GNU Unifont - -#Read sfdfile for character widths -function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}()) - state=:seekchar - lineno = 0 - codepoint = width = nothing - for line in readlines(open(filename)) - lineno += 1 - if state==:seekchar #StartChar: nonmarkingreturn - if occursin("StartChar: ", line) - codepoint = nothing - width = nothing - state = :readdata - end - elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024 - occursin("Encoding:", line) && (codepoint = parse(Int, split(line)[3])) - occursin("Width:", line) && (width = parse(Int, split(line)[2])) - if codepoint!=nothing && width!=nothing && codepoint >= 0 - w=div(width, 512) # 512 units to the en - if w > 0 - # only add nonzero widths, since (1) the default is zero - # and (2) this circumvents some apparent bugs in Unifont - # (https://savannah.gnu.org/bugs/index.php?45395) - CharWidths[codepoint] = w - end - state = :seekchar - end - end - end - CharWidths -end -CharWidths=parsesfd("unifont.sfd", CharWidths) -CharWidths=parsesfd("unifont_upper.sfd", CharWidths) - -############################################################################# # Widths from UAX #11: East Asian Width -# .. these take precedence over the Unifont width for all codepoints +# .. these take precedence for all codepoints # listed explicitly as wide/full/narrow/half-width for line in readlines(open("EastAsianWidth.txt")) @@ -180,16 +146,6 @@ CharWidths[0x00ad]=1 CharWidths[0x2028]=0 CharWidths[0x2029]=0 -#By definition, should be narrow = width of 1 en space -#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/ -CharWidths[0x202f]=1 - -#By definition, should be wide = width of 1 em space -#0x002001 ' ' category: Zs name: EM QUAD/ -#0x002003 ' ' category: Zs name: EM SPACE/ -CharWidths[0x2001]=2 -CharWidths[0x2003]=2 - ############################################################################# # Output (to a file or pipe) for processing by data_generator.rb, # encoded as a sequence of intervals. |