uline: Add unicode line reader

1 files changed, 386 insertions, 0 deletions

diff --git a/test/uline.cc b/test/uline.cc
new file mode 100644
index 0000000..ca3f2bb
--- /dev/null
+++ b/test/uline.cc
@@ -0,0 +1,386 @@
+#include "uline.hh"
+
+#include "io_test_helper.hh"
+
+#include <cstddef>
+#include <gtest/gtest.h>
+#include <limits>
+#include <string_view>
+#include <utility>
+
+using namespace std::literals::string_view_literals;
+
+TEST(u8line, empty) {
+  auto reader = u8::line::open(u8::open(io::memory("")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(0, reader->number());
+}
+
+TEST(u8line, one_line) {
+  auto reader =
+      u8::line::open(u8::open(io::memory("r\xc3\xa4ksm\xc3\xb6rg\xc3\xa5s")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("r\xc3\xa4ksm\xc3\xb6rg\xc3\xa5s", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(1, reader->number());
+}
+
+TEST(u8line, many_lines) {
+  auto reader = u8::line::open(u8::open(io::memory("foo\nbar\nfoobar\n")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foo", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("bar", line.value());
+  EXPECT_EQ(2, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foobar", line.value());
+  EXPECT_EQ(3, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(3, reader->number());
+}
+
+TEST(u8line, many_lines_mixed) {
+  auto reader = u8::line::open(u8::open(io::memory("foo\r\nbar\rfoobar\n")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foo", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("bar", line.value());
+  EXPECT_EQ(2, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foobar", line.value());
+  EXPECT_EQ(3, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(3, reader->number());
+}
+
+TEST(u8line, empty_line) {
+  auto reader = u8::line::open(u8::open(io::memory("\n")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(1, reader->number());
+}
+
+TEST(u8line, max_line) {
+  auto reader = u8::line::open(
+      u8::open(io::memory("012345678901234567890123456789")), 10);
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("0123456789", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("0123456789", line.value());
+  EXPECT_EQ(2, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("0123456789", line.value());
+  EXPECT_EQ(3, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(3, reader->number());
+}
+
+TEST(u8line, read_error) {
+  auto reader = u8::line::open(u8::open(
+      io_make_breaking(io::memory("foo bar fum\nfim zam"), /* offset */ 5)));
+  auto line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_FALSE(line.error().eof);
+  EXPECT_EQ(io::ReadError::Error, line.error().io_error.value());
+}
+
+TEST(u8line, read_error_newline) {
+  auto reader = u8::line::open(u8::open(
+      io_make_breaking(io::memory("foo bar\r\nfim zam"), /* offset */ 8)));
+  auto line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_FALSE(line.error().eof);
+  EXPECT_EQ(io::ReadError::Error, line.error().io_error.value());
+}
+
+TEST(u8line, blocky) {
+  auto reader = u8::line::open(
+      u8::open(io_make_max_block(io::memory("foo bar\r\nfim zam"),
+                                 /* max_block_size */ 1)));
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foo bar", line.value());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("fim zam", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u8line, blocky_newline) {
+  auto reader = u8::line::open(
+      u8::open(io_make_max_block(io::memory("foo bar\r\nfim zam"),
+                                 /* max_block_size */ 8)));
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foo bar", line.value());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("fim zam", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u8line, eof_newline) {
+  auto reader = u8::line::open(u8::open(io::memory("foo bar\r")));
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foo bar", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u8line, max_newline) {
+  auto reader = u8::line::open(u8::open(io::memory("foo bar\r")), 6);
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("foo ba", line.value());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ("r", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u8line, max_line_overflow) {
+  EXPECT_DEATH_IF_SUPPORTED(
+      {
+        std::ignore = u8::line::open(u8::open(io::memory("")),
+                                     std::numeric_limits<size_t>::max());
+      },
+      "");
+}
+
+TEST(u16line, empty) {
+  auto reader = u16::line::open(u16::open(io::memory("")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(0, reader->number());
+}
+
+TEST(u16line, one_line) {
+  auto reader =
+      u16::line::open(u16::open(io::memory("r\xc3\xa4ksm\xc3\xb6rg\xc3\xa5s")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"r\u00e4ksm\u00f6rg\u00e5s", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(1, reader->number());
+}
+
+TEST(u16line, one_line_u16) {
+  auto u16tmp = u"r\u00e4ksm\u00f6rg\u00e5s"sv;
+  auto reader = u16::line::open(u16::open(io::memory(std::string(
+      reinterpret_cast<const char*>(u16tmp.data()), u16tmp.size() * 2))));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"r\u00e4ksm\u00f6rg\u00e5s", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(1, reader->number());
+}
+
+TEST(u16line, many_lines) {
+  auto reader = u16::line::open(u16::open(io::memory("foo\nbar\nfoobar\n")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foo", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"bar", line.value());
+  EXPECT_EQ(2, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foobar", line.value());
+  EXPECT_EQ(3, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(3, reader->number());
+}
+
+TEST(u16line, many_lines_mixed) {
+  auto reader = u16::line::open(u16::open(io::memory("foo\r\nbar\rfoobar\n")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foo", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"bar", line.value());
+  EXPECT_EQ(2, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foobar", line.value());
+  EXPECT_EQ(3, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(3, reader->number());
+}
+
+TEST(u16line, empty_line) {
+  auto reader = u16::line::open(u16::open(io::memory("\n")));
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(1, reader->number());
+}
+
+TEST(u16line, max_line) {
+  auto reader = u16::line::open(
+      u16::open(io::memory("012345678901234567890123456789")), 10);
+  EXPECT_EQ(0, reader->number());
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"0123456789", line.value());
+  EXPECT_EQ(1, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"0123456789", line.value());
+  EXPECT_EQ(2, reader->number());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"0123456789", line.value());
+  EXPECT_EQ(3, reader->number());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+  EXPECT_EQ(3, reader->number());
+}
+
+TEST(u16line, read_error) {
+  auto reader = u16::line::open(u16::open(
+      io_make_breaking(io::memory("foo bar fum\nfim zam"), /* offset */ 5)));
+  auto line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_FALSE(line.error().eof);
+  EXPECT_EQ(io::ReadError::Error, line.error().io_error.value());
+}
+
+TEST(u16line, read_error_newline) {
+  auto reader = u16::line::open(u16::open(
+      io_make_breaking(io::memory("foo bar\r\nfim zam"), /* offset */ 8)));
+  auto line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_FALSE(line.error().eof);
+  EXPECT_EQ(io::ReadError::Error, line.error().io_error.value());
+}
+
+TEST(u16line, blocky) {
+  auto reader = u16::line::open(
+      u16::open(io_make_max_block(io::memory("foo bar\r\nfim zam"),
+                                  /* max_block_size */ 1)));
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foo bar", line.value());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"fim zam", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u16line, blocky_newline) {
+  auto reader = u16::line::open(
+      u16::open(io_make_max_block(io::memory("foo bar\r\nfim zam"),
+                                  /* max_block_size */ 8)));
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foo bar", line.value());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"fim zam", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u16line, eof_newline) {
+  auto reader = u16::line::open(u16::open(io::memory("foo bar\r")));
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foo bar", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u16line, max_newline) {
+  auto reader = u16::line::open(u16::open(io::memory("foo bar\r")), 6);
+  auto line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"foo ba", line.value());
+  line = reader->read();
+  ASSERT_TRUE(line.has_value());
+  EXPECT_EQ(u"r", line.value());
+  line = reader->read();
+  ASSERT_FALSE(line.has_value());
+  EXPECT_TRUE(line.error().eof);
+}
+
+TEST(u16line, max_line_overflow) {
+  EXPECT_DEATH_IF_SUPPORTED(
+      {
+        std::ignore = u16::line::open(u16::open(io::memory("")),
+                                      std::numeric_limits<size_t>::max());
+      },
+      "");
+}