Travel3: Simple image and video display site

Reads the images and videos from filesystem and builds a site in
memroy.

1 files changed, 556 insertions, 0 deletions

diff --git a/src/travel.cc b/src/travel.cc
new file mode 100644
index 0000000..f8adf50
--- /dev/null
+++ b/src/travel.cc
@@ -0,0 +1,556 @@
+#include "common.hh"
+
+#include "config.hh"
+#include "files_finder.hh"
+#include "image.hh"
+#include "logger.hh"
+#include "rotation.hh"
+#include "strutil.hh"
+#include "task_runner.hh"
+#include "timezone.hh"
+#include "travel.hh"
+#include "video.hh"
+#include "weak_ptr.hh"
+
+#include <algorithm>
+#include <mutex>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+class TravelImpl : public Travel, public FilesFinder::Delegate {
+public:
+  TravelImpl(std::shared_ptr<Logger> logger,
+             std::shared_ptr<TaskRunner> runner)
+    : logger_(std::move(logger)), runner_(std::move(runner)),
+      weak_ptr_owner_(this) {}
+
+  bool setup(Logger* logger, Config* config) override {
+    root_ = config->get_path("site.root", "");
+    if (root_.empty()) {
+      logger->err("site.root must be set to the root directory for travels.");
+      return false;
+    }
+    auto travel_files_threads = config->get("workers", 4);
+    if (!travel_files_threads.has_value()) {
+      logger->err("workers is unknown value: '%s'",
+                  config->get("workers", nullptr));
+      return false;
+    }
+    if (travel_files_threads.value() <= 0) {
+      logger->err("workers must be > 0");
+      return false;
+    }
+    timezone_ = Timezone::create(logger_,
+                                 config->get_path("geojson.database", ""),
+                                 config->get_path("zoneinfo.root",
+                                                  "/usr/share/zoneinfo"));
+    worker_threads_ = travel_files_threads.value();
+    return true;
+  }
+
+  void start() override {
+    workers_ = TaskRunner::create(worker_threads_);
+    finder_ = FilesFinder::create(logger_, workers_, root_, this,
+                                  worker_threads_);
+  }
+
+  void reload() override {
+    // Make implementation simpler by ignoring calls to reload() while
+    // previoys start() or reload() is still in progress.
+    if (finder_) {
+      logger_->warn("Reload called while still loading, ignored.");
+      return;
+    }
+
+    workers_.reset();
+    trips_.clear();
+    trip_index_.clear();
+
+    start();
+  }
+
+  size_t trips() const override {
+    return trips_.size();
+  }
+
+  Trip const& trip(size_t i) const override {
+    assert(i < trips_.size());
+    return trips_[i];
+  }
+
+  void call_when_loaded(std::function<void()> callback) override {
+    if (finder_) {
+      call_when_loaded_.push_back(std::move(callback));
+    } else {
+      callback();
+    }
+  }
+
+  // Called on any thread.
+  bool include_dir(std::string_view name, uint16_t depth) const override {
+    if (depth > 0)
+      return FilesFinder::Delegate::include_dir(name, depth);
+    return valid_trip_id(name);
+  }
+
+  // Runs on workers
+  void file(std::filesystem::path path) override {
+    std::string media_id;
+    std::string trip_id = get_trip_id(path, &media_id);
+    if (trip_id.empty()) {
+      logger_->warn("Ignoring %s because no trip id found", path.c_str());
+      return;
+    }
+    auto image = get_image_info(media_id, path);
+    if (image) {
+      if (image.value().date_.empty()) {
+        logger_->dbg("Ignoring %s, image without a date.", path.c_str());
+        return;
+      }
+      runner_->post(std::bind(&TravelImpl::weak_finder_image,
+                              weak_ptr_owner_.get(),
+                              trip_id,
+                              image.value()));
+      return;
+    }
+    auto video = get_video_info(media_id, path, timezone_.get());
+    if (video) {
+      if (video.value().date_.empty()) {
+        logger_->dbg("Ignoring %s, video without a date.", path.c_str());
+        return;
+      }
+      runner_->post(std::bind(&TravelImpl::weak_finder_video,
+                              weak_ptr_owner_.get(),
+                              trip_id,
+                              video.value()));
+      return;
+    }
+    logger_->dbg("Ignoring %s, not an image or a video.", path.c_str());
+  }
+
+  // Runs on workers
+  void done() override {
+    runner_->post(std::bind(&TravelImpl::weak_finder_done,
+                            weak_ptr_owner_.get()));
+  }
+
+private:
+  struct Info {
+    std::string id_;
+    std::filesystem::path path_;
+    uint64_t width_;
+    uint64_t height_;
+    Location location_;
+    Date date_;
+
+    Info(std::string id, std::filesystem::path path,
+         uint64_t width, uint64_t height, Location location, Date date)
+      : id_(std::move(id)), path_(std::move(path)),
+        width_(width), height_(height),
+        location_(location), date_(date) {}
+  };
+
+  struct ImageInfo : Info {
+    Rotation rotation_;
+    std::string thumbnail_mime_type_;
+    uint64_t thumbnail_size_;
+
+    ImageInfo(std::string id, std::filesystem::path path,
+              uint64_t width, uint64_t height,
+              Location location, Date date, Rotation rotation,
+              std::string thumbnail_mime_type, uint64_t thumbnail_size)
+      : Info(std::move(id), std::move(path), width, height, location,
+             date), rotation_(rotation),
+        thumbnail_mime_type_(std::move(thumbnail_mime_type)),
+        thumbnail_size_(thumbnail_size) {}
+  };
+
+  struct VideoInfo : Info {
+    double length_;
+
+    VideoInfo(std::string id, std::filesystem::path path,
+              uint64_t width, uint64_t height,
+              Location location, Date date, double length)
+      : Info(std::move(id), std::move(path), width, height, location,
+             date), length_(length) {}
+  };
+
+  class MediaImpl : public virtual Media, public virtual Thumbnail {
+  public:
+    explicit MediaImpl(ImageInfo info)
+      : image_(std::move(info)) {}
+
+    explicit MediaImpl(VideoInfo info)
+      : video_(std::move(info)) {}
+
+    std::string_view id() const override {
+      return image_ ? image_->id_ : video_->id_;
+    }
+
+    std::filesystem::path const& path() const override {
+      return image_ ? image_->path_ : video_->path_;
+    }
+
+    Type type() const override {
+      return image_ ? Type::IMAGE : Type::VIDEO;
+    }
+
+    uint64_t width() const override {
+      return image_ ? image_->width_ : video_->width_;
+    }
+    uint64_t height() const override {
+      return image_ ? image_->height_ : video_->height_;
+    }
+
+    Location location() const override {
+      return image_ ? image_->location_ : video_->location_;
+    }
+
+    Date date() const override {
+      return image_ ? image_->date_ : video_->date_;
+    }
+
+    double length() const override {
+      return image_ ? 0.0 : video_->length_;
+    }
+
+    Rotation rotation() const override {
+      return image_ ? image_->rotation_ : Rotation::UNKNOWN;
+    }
+
+    Thumbnail const* thumbnail() const override {
+      return image_ && !image_->thumbnail_mime_type_.empty() &&
+        image_->thumbnail_size_ > 0 ? this : nullptr;
+    }
+
+    ThumbType thumb_type() const override {
+      return ThumbType::EXIF;
+    }
+
+    std::string_view mime_type() const override {
+      return image_->thumbnail_mime_type_;
+    }
+
+    uint64_t size() const override {
+      return image_->thumbnail_size_;
+    }
+
+  private:
+    std::optional<ImageInfo> image_;
+    std::optional<VideoInfo> video_;
+  };
+
+  class DayImpl : public Day {
+  public:
+    DayImpl(Date day, size_t first)
+      : day_(day), first_(first), last_(first) {
+    }
+
+    Date date() const override {
+      return day_;
+    }
+
+    size_t first() const override {
+      return first_;
+    }
+
+    size_t last() const override {
+      return last_;
+    }
+
+    void increment_last() {
+      ++last_;
+    }
+
+  private:
+    Date const day_;
+    size_t const first_;
+    size_t last_;
+  };
+
+  class TripImpl : public Trip {
+  public:
+    TripImpl(std::string id, std::string name, uint16_t year)
+      : id_(std::move(id)), name_(std::move(name)), year_(year) {}
+
+    std::string_view id() const override {
+      return id_;
+    }
+
+    std::string_view title() const override {
+      return name_;
+    }
+
+    uint16_t year() const override {
+      return year_;
+    }
+
+    Location location() const override {
+      return location_;
+    }
+
+    uint64_t images() const override {
+      return images_;
+    }
+
+    uint64_t videos() const override {
+      return videos_;
+    }
+
+    size_t media_count() const override {
+      return media_.size();
+    }
+
+    Media const& media(size_t i) const override {
+      assert(i < media_.size());
+      return media_[i];
+    }
+
+    size_t day_count() const override {
+      return day_.size();
+    }
+
+    Day const& day(size_t i) const override {
+      assert(i < day_.size());
+      return day_[i];
+    }
+
+    void add_image(ImageInfo info) {
+      ++images_;
+      media_.emplace_back(std::move(info));
+    }
+
+    void add_video(VideoInfo info) {
+      ++videos_;
+      media_.emplace_back(std::move(info));
+    }
+
+    void sort_media() {
+      std::sort(media_.begin(), media_.end(),
+                [] (MediaImpl const& a, MediaImpl const& b) {
+                  return a.date() < b.date();
+                });
+    }
+
+    void setup_days() {
+      for (size_t i = 0; i < media_.size(); ++i) {
+        auto day = media_[i].date().day();
+        if (day_.empty() || day != day_.back().date()) {
+          assert(day_.empty() || day > day_.back().date());
+          day_.emplace_back(day, i);
+        } else {
+          assert(i == day_.back().last() + 1);
+          day_.back().increment_last();
+        }
+      }
+    }
+
+    void set_location(Location location) {
+      location_ = location;
+    }
+
+  private:
+    std::string id_;
+    std::string name_;
+    uint16_t year_;
+    Location location_;
+    uint64_t images_{0};
+    uint64_t videos_{0};
+    std::vector<MediaImpl> media_;
+    std::vector<DayImpl> day_;
+  };
+
+  // Called from workers
+  std::string get_trip_id(std::filesystem::path const& path,
+                          std::string* out_child_id) const {
+    std::string child_id;
+    std::filesystem::path tmp = path;
+    while (true) {
+      if (!tmp.has_parent_path())
+        break;
+      auto parent = tmp.parent_path();
+      if (parent == root_) {
+        if (out_child_id)
+          *out_child_id = std::move(child_id);
+        return tmp.filename();
+      }
+      if (!child_id.empty())
+        child_id.insert(0, "/");
+      child_id.insert(0, tmp.filename());
+      tmp = parent;
+    }
+    return std::string();
+  }
+
+  static void weak_finder_image(std::shared_ptr<WeakPtr<TravelImpl>> weak_ptr,
+                                std::string const& trip_id,
+                                ImageInfo const& info) {
+    auto* ptr = weak_ptr->get();
+    if (ptr)
+      ptr->finder_image(trip_id, info);
+  }
+
+  static void weak_finder_video(std::shared_ptr<WeakPtr<TravelImpl>> weak_ptr,
+                                std::string const& trip_id,
+                                VideoInfo const& info) {
+    auto* ptr = weak_ptr->get();
+    if (ptr)
+      ptr->finder_video(trip_id, info);
+  }
+
+  static void weak_finder_done(std::shared_ptr<WeakPtr<TravelImpl>> weak_ptr) {
+    auto* ptr = weak_ptr->get();
+    if (ptr)
+      ptr->finder_done();
+  }
+
+  TripImpl* get_trip(std::string const& trip_id) {
+    auto it = trip_index_.find(trip_id);
+    if (it != trip_index_.end())
+      return &trips_[it->second];
+
+    std::string name;
+    uint16_t year;
+    if (parse_trip_id(trip_id, &name, &year)) {
+      auto index = trips_.size();
+      trips_.emplace_back(trip_id, std::move(name), year);
+      trip_index_[trip_id] = index;
+      return &trips_[index];
+    } else {
+      // include_dir should make sure of this doesn't happen.
+      assert(false);
+      return nullptr;
+    }
+  }
+
+  void finder_image(std::string const& trip_id,
+                    ImageInfo const& info) {
+    auto* trip_ptr = get_trip(trip_id);
+    if (trip_ptr)
+      trip_ptr->add_image(info);
+  }
+
+  void finder_video(std::string const& trip_id,
+                    VideoInfo const& info) {
+    auto* trip_ptr = get_trip(trip_id);
+    if (trip_ptr)
+      trip_ptr->add_video(info);
+  }
+
+  void finder_done() {
+    finder_.reset();
+
+    for (auto& trip_impl : trips_)
+      cleanup_trip(trip_impl);
+
+    logger_->info("Trips all loaded.");
+
+    for (auto& callback : call_when_loaded_) {
+      callback();
+    }
+    call_when_loaded_.clear();
+  }
+
+  void cleanup_trip(TripImpl& trip_impl) {
+    // Sort by date
+    trip_impl.sort_media();
+
+    trip_impl.setup_days();
+
+    // TODO: Remove outliers
+    // TODO: Use some weighted median instead of average
+    Location loc;
+    size_t count = 0;
+    for (size_t i = 0; i < trip_impl.media_count(); ++i) {
+      auto& media = trip_impl.media(i);
+      if (media.location().empty())
+        continue;
+      if (loc.empty()) {
+        loc = media.location();
+        count = 1;
+      } else {
+        loc.lat += media.location().lat;
+        loc.lng += media.location().lng;
+        ++count;
+      }
+    }
+    loc.lat /= count;
+    loc.lng /= count;
+    trip_impl.set_location(loc);
+  }
+
+  static std::optional<ImageInfo> get_image_info(std::string id,
+                                                 std::filesystem::path path) {
+    auto image = Image::load(path);
+    if (image) {
+      auto* thumb = image->thumbnail();
+      return ImageInfo(std::move(id), std::move(path),
+                       image->width(), image->height(),
+                       image->location(), image->date(), image->rotation(),
+                       thumb ? std::string(thumb->mime_type()) : std::string(),
+                       thumb ? thumb->size() : 0);
+    }
+    return std::nullopt;
+  }
+
+  static std::optional<VideoInfo> get_video_info(std::string id,
+                                                 std::filesystem::path path,
+                                                 Timezone const* timezone) {
+    auto video = Video::load(path, timezone);
+    if (video)
+      return VideoInfo(std::move(id), std::move(path),
+                       video->width(), video->height(),
+                       video->location(), video->date(), video->length());
+    return std::nullopt;
+  }
+
+  static bool valid_trip_id(std::string_view id) {
+    return parse_trip_id(id, nullptr, nullptr);
+  }
+
+  static bool parse_trip_id(std::string_view id,
+                            std::string* name, uint16_t* year) {
+    auto it = id.find('-');
+    if (it == std::string_view::npos)
+      return false;
+    if (name)
+      name->assign(str::trim(id.substr(0, it)));
+    auto tmp = str::parse_uint16(std::string(id, it + 1, std::string::npos));
+    if (!tmp)
+      return false;
+    if (*tmp < 1950)
+      return false;
+    if (year)
+      *year = *tmp;
+    return true;
+  }
+
+  std::shared_ptr<Logger> logger_;
+  std::shared_ptr<TaskRunner> runner_;
+  std::unique_ptr<Timezone> timezone_;
+  size_t worker_threads_{0};
+  std::filesystem::path root_;
+  std::vector<TripImpl> trips_;
+  std::unordered_map<std::string, size_t> trip_index_;
+  std::vector<std::function<void()>> call_when_loaded_;
+
+  std::mutex worker_mutex_;
+
+  // It is important that workers_ is (next to) last as it blocks leftover
+  // workers in destructor so should be destroyed first.
+  std::shared_ptr<TaskRunner> workers_;
+  // finder depends on workers so must be destroyed before.
+  std::unique_ptr<FilesFinder> finder_;
+  WeakPtrOwner<TravelImpl> weak_ptr_owner_;
+};
+
+}  // namespace
+
+std::unique_ptr<Travel> Travel::create(std::shared_ptr<Logger> logger,
+                                       std::shared_ptr<TaskRunner> runner) {
+  return std::make_unique<TravelImpl>(std::move(logger), std::move(runner));
+}
+