From 18a622f378b403788c67fc785d30f4609caa3fc7 Mon Sep 17 00:00:00 2001 From: Joel Klinghed Date: Mon, 15 Sep 2025 20:52:51 +0200 Subject: uio: Unicode reader Reads UTF-8 and UTF-16 into UTF-8 or UTF-16 strings. If strict is true, fails at first invalid character. If strict is false, invalid characters are replaced with U+FFFD. For the replacement, I changed behavior if uN::read_replace to only jump one byte. Otherwise a common invalid case when ISO-8859-1 or WIN-1252 are read as UTF-8 would skip many characters. If skip_bom is true any bom at start of stream is ignored. If skip_bom is false any bom will be included. Input format can be forced, if not detect is used which will try to guess and then fallback to UTF-8. --- src/io.cc | 8 -------- 1 file changed, 8 deletions(-) (limited to 'src/io.cc') diff --git a/src/io.cc b/src/io.cc index baf162a..e0ab787 100644 --- a/src/io.cc +++ b/src/io.cc @@ -149,10 +149,6 @@ class StringReader : public MemoryReader { } // namespace -std::expected Reader::read(std::string& str) { - return read(str.data(), str.size()); -} - std::expected Reader::repeat_read(void* dst, size_t max) { auto ret = read(dst, max); if (!ret.has_value() || ret.value() == 0 || ret.value() == max) @@ -171,10 +167,6 @@ std::expected Reader::repeat_read(void* dst, size_t max) { return offset; } -std::expected Reader::repeat_read(std::string& str) { - return repeat_read(str.data(), str.size()); -} - std::expected Reader::repeat_skip(size_t max) { auto ret = skip(max); if (!ret.has_value() || ret.value() == 0 || ret.value() == max) -- cgit v1.3