1-- $Id: testes/utf8.lua $
  2-- See Copyright Notice in file all.lua
  3
  4-- UTF-8 file
  5
  6print "testing UTF-8 library"
  7
  8local utf8 = require'utf8'
  9
 10
 11local function checkerror (msg, f, ...)
 12  local s, err = pcall(f, ...)
 13  assert(not s and string.find(err, msg))
 14end
 15
 16
 17local function len (s)
 18  return #string.gsub(s, "[\x80-\xBF]", "")
 19end
 20
 21
 22local justone = "^" .. utf8.charpattern .. "$"
 23
 24-- 't' is the list of codepoints of 's'
 25local function checksyntax (s, t)
 26  -- creates a string "return '\u{t[1]}...\u{t[n]}'"
 27  local ts = {"return '"}
 28  for i = 1, #t do ts[i + 1] = string.format("\\u{%x}", t[i]) end
 29  ts[#t + 2] = "'"
 30  ts = table.concat(ts)
 31  -- its execution should result in 's'
 32  assert(assert(load(ts))() == s)
 33end
 34
 35assert(not utf8.offset("alo", 5))
 36assert(not utf8.offset("alo", -4))
 37
 38-- 'check' makes several tests over the validity of string 's'.
 39-- 't' is the list of codepoints of 's'.
 40local function check (s, t, nonstrict)
 41  local l = utf8.len(s, 1, -1, nonstrict)
 42  assert(#t == l and len(s) == l)
 43  assert(utf8.char(table.unpack(t)) == s)   -- 't' and 's' are equivalent
 44
 45  assert(utf8.offset(s, 0) == 1)
 46
 47  checksyntax(s, t)
 48
 49  -- creates new table with all codepoints of 's'
 50  local t1 = {utf8.codepoint(s, 1, -1, nonstrict)}
 51  assert(#t == #t1)
 52  for i = 1, #t do assert(t[i] == t1[i]) end   -- 't' is equal to 't1'
 53
 54  for i = 1, l do   -- for all codepoints
 55    local pi = utf8.offset(s, i)        -- position of i-th char
 56    local pi1 = utf8.offset(s, 2, pi)   -- position of next char
 57    assert(string.find(string.sub(s, pi, pi1 - 1), justone))
 58    assert(utf8.offset(s, -1, pi1) == pi)
 59    assert(utf8.offset(s, i - l - 1) == pi)
 60    assert(pi1 - pi == #utf8.char(utf8.codepoint(s, pi, pi, nonstrict)))
 61    for j = pi, pi1 - 1 do
 62      assert(utf8.offset(s, 0, j) == pi)
 63    end
 64    for j = pi + 1, pi1 - 1 do
 65      assert(not utf8.len(s, j))
 66    end
 67   assert(utf8.len(s, pi, pi, nonstrict) == 1)
 68   assert(utf8.len(s, pi, pi1 - 1, nonstrict) == 1)
 69   assert(utf8.len(s, pi, -1, nonstrict) == l - i + 1)
 70   assert(utf8.len(s, pi1, -1, nonstrict) == l - i)
 71   assert(utf8.len(s, 1, pi, nonstrict) == i)
 72  end
 73
 74  local i = 0
 75  for p, c in utf8.codes(s, nonstrict) do
 76    i = i + 1
 77    assert(c == t[i] and p == utf8.offset(s, i))
 78    assert(utf8.codepoint(s, p, p, nonstrict) == c)
 79  end
 80  assert(i == #t)
 81
 82  i = 0
 83  for c in string.gmatch(s, utf8.charpattern) do
 84    i = i + 1
 85    assert(c == utf8.char(t[i]))
 86  end
 87  assert(i == #t)
 88
 89  for i = 1, l do
 90    assert(utf8.offset(s, i) == utf8.offset(s, i - l - 1, #s + 1))
 91  end
 92
 93end
 94
 95
 96do    -- error indication in utf8.len
 97  local function check (s, p)
 98    local a, b = utf8.len(s)
 99    assert(not a and b == p)
100  end
101  check("abc\xE3def", 4)
102  check("\xF4\x9F\xBF", 1)
103  check("\xF4\x9F\xBF\xBF", 1)
104  -- spurious continuation bytes
105  check("汉字\x80", #("汉字") + 1)
106  check("\x80hello", 1)
107  check("hel\x80lo", 4)
108  check("汉字\xBF", #("汉字") + 1)
109  check("\xBFhello", 1)
110  check("hel\xBFlo", 4)
111end
112
113-- errors in utf8.codes
114do
115  local function errorcodes (s)
116    checkerror("invalid UTF%-8 code",
117      function ()
118        for c in utf8.codes(s) do assert(c) end
119      end)
120  end
121  errorcodes("ab\xff")
122  errorcodes("\u{110000}")
123  errorcodes("in\x80valid")
124  errorcodes("\xbfinvalid")
125  errorcodes("αλφ\xBFα")
126
127  -- calling interation function with invalid arguments
128  local f = utf8.codes("")
129  assert(f("", 2) == nil)
130  assert(f("", -1) == nil)
131  assert(f("", math.mininteger) == nil)
132
133end
134
135-- error in initial position for offset
136checkerror("position out of bounds", utf8.offset, "abc", 1, 5)
137checkerror("position out of bounds", utf8.offset, "abc", 1, -4)
138checkerror("position out of bounds", utf8.offset, "", 1, 2)
139checkerror("position out of bounds", utf8.offset, "", 1, -1)
140checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
141checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
142checkerror("continuation byte", utf8.offset, "\x80", 1)
143
144-- error in indices for len
145checkerror("out of bounds", utf8.len, "abc", 0, 2)
146checkerror("out of bounds", utf8.len, "abc", 1, 4)
147
148
149local s = "hello World"
150local t = {string.byte(s, 1, -1)}
151for i = 1, utf8.len(s) do assert(t[i] == string.byte(s, i)) end
152check(s, t)
153
154check("汉字/漢字", {27721, 23383, 47, 28450, 23383,})
155
156do
157  local s = "áéí\128"
158  local t = {utf8.codepoint(s,1,#s - 1)}
159  assert(#t == 3 and t[1] == 225 and t[2] == 233 and t[3] == 237)
160  checkerror("invalid UTF%-8 code", utf8.codepoint, s, 1, #s)
161  checkerror("out of bounds", utf8.codepoint, s, #s + 1)
162  t = {utf8.codepoint(s, 4, 3)}
163  assert(#t == 0)
164  checkerror("out of bounds", utf8.codepoint, s, -(#s + 1), 1)
165  checkerror("out of bounds", utf8.codepoint, s, 1, #s + 1)
166  -- surrogates
167  assert(utf8.codepoint("\u{D7FF}") == 0xD800 - 1)
168  assert(utf8.codepoint("\u{E000}") == 0xDFFF + 1)
169  assert(utf8.codepoint("\u{D800}", 1, 1, true) == 0xD800)
170  assert(utf8.codepoint("\u{DFFF}", 1, 1, true) == 0xDFFF)
171  assert(utf8.codepoint("\u{7FFFFFFF}", 1, 1, true) == 0x7FFFFFFF)
172end
173
174assert(utf8.char() == "")
175assert(utf8.char(0, 97, 98, 99, 1) == "\0abc\1")
176
177assert(utf8.codepoint(utf8.char(0x10FFFF)) == 0x10FFFF)
178assert(utf8.codepoint(utf8.char(0x7FFFFFFF), 1, 1, true) == (1<<31) - 1)
179
180checkerror("value out of range", utf8.char, 0x7FFFFFFF + 1)
181checkerror("value out of range", utf8.char, -1)
182
183local function invalid (s)
184  checkerror("invalid UTF%-8 code", utf8.codepoint, s)
185  assert(not utf8.len(s))
186end
187
188-- UTF-8 representation for 0x11ffff (value out of valid range)
189invalid("\xF4\x9F\xBF\xBF")
190
191-- surrogates
192invalid("\u{D800}")
193invalid("\u{DFFF}")
194
195-- overlong sequences
196invalid("\xC0\x80")          -- zero
197invalid("\xC1\xBF")          -- 0x7F (should be coded in 1 byte)
198invalid("\xE0\x9F\xBF")      -- 0x7FF (should be coded in 2 bytes)
199invalid("\xF0\x8F\xBF\xBF")  -- 0xFFFF (should be coded in 3 bytes)
200
201
202-- invalid bytes
203invalid("\x80")  -- continuation byte
204invalid("\xBF")  -- continuation byte
205invalid("\xFE")  -- invalid byte
206invalid("\xFF")  -- invalid byte
207
208
209-- empty string
210check("", {})
211
212-- minimum and maximum values for each sequence size
213s = "\0 \x7F\z
214     \xC2\x80 \xDF\xBF\z
215     \xE0\xA0\x80 \xEF\xBF\xBF\z
216     \xF0\x90\x80\x80  \xF4\x8F\xBF\xBF"
217s = string.gsub(s, " ", "")
218check(s, {0,0x7F, 0x80,0x7FF, 0x800,0xFFFF, 0x10000,0x10FFFF})
219
220do
221  -- original UTF-8 values
222  local s = "\u{4000000}\u{7FFFFFFF}"
223  assert(#s == 12)
224  check(s, {0x4000000, 0x7FFFFFFF}, true)
225
226  s = "\u{200000}\u{3FFFFFF}"
227  assert(#s == 10)
228  check(s, {0x200000, 0x3FFFFFF}, true)
229
230  s = "\u{10000}\u{1fffff}"
231  assert(#s == 8)
232  check(s, {0x10000, 0x1FFFFF}, true)
233end
234
235local x = "日本語a-4\0éó"
236check(x, {26085, 26412, 35486, 97, 45, 52, 0, 233, 243})
237
238
239-- Supplementary Characters
240check("𣲷𠜎𠱓𡁻𠵼ab𠺢",
241      {0x23CB7, 0x2070E, 0x20C53, 0x2107B, 0x20D7C, 0x61, 0x62, 0x20EA2,})
242
243check("𨳊𩶘𦧺𨳒𥄫𤓓\xF4\x8F\xBF\xBF",
244      {0x28CCA, 0x29D98, 0x269FA, 0x28CD2, 0x2512B, 0x244D3, 0x10ffff})
245
246
247local i = 0
248for p, c in string.gmatch(x, "()(" .. utf8.charpattern .. ")") do
249  i = i + 1
250  assert(utf8.offset(x, i) == p)
251  assert(utf8.len(x, p) == utf8.len(x) - i + 1)
252  assert(utf8.len(c) == 1)
253  for j = 1, #c - 1 do
254    assert(utf8.offset(x, 0, p + j - 1) == p)
255  end
256end
257
258print'ok'
259