From 18a622f378b403788c67fc785d30f4609caa3fc7 Mon Sep 17 00:00:00 2001 From: Joel Klinghed Date: Mon, 15 Sep 2025 20:52:51 +0200 Subject: uio: Unicode reader Reads UTF-8 and UTF-16 into UTF-8 or UTF-16 strings. If strict is true, fails at first invalid character. If strict is false, invalid characters are replaced with U+FFFD. For the replacement, I changed behavior if uN::read_replace to only jump one byte. Otherwise a common invalid case when ISO-8859-1 or WIN-1252 are read as UTF-8 would skip many characters. If skip_bom is true any bom at start of stream is ignored. If skip_bom is false any bom will be included. Input format can be forced, if not detect is used which will try to guess and then fallback to UTF-8. --- src/gen_ugc.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/gen_ugc.cc') diff --git a/src/gen_ugc.cc b/src/gen_ugc.cc index e9bce11..7583272 100644 --- a/src/gen_ugc.cc +++ b/src/gen_ugc.cc @@ -149,6 +149,8 @@ std::string_view ioerr2str(io::ReadError error) { return "Invalid (compressed) data"; case io::ReadError::Error: return "Fatal error"; + case io::ReadError::MaxTooSmall: + return "Too small buffer"; } std::unreachable(); } -- cgit v1.3