Initial support for translation reviews

1 files changed, 288 insertions, 24 deletions

diff --git a/server/src/trans.rs b/server/src/trans.rs
index 6a18e27..c4e3b45 100644
--- a/server/src/trans.rs
+++ b/server/src/trans.rs
@@ -1,27 +1,46 @@
+#![allow(dead_code)]
+
 use anyhow;
+use futures::stream::TryStreamExt;
+use rocket_db_pools::{sqlx, Database, Pool};
+use sorted_insert::SortedInsertByKey;
+use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
+use std::fs::File;
+use std::io::{BufReader, Read};
 use std::iter::{repeat, IntoIterator};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use tokio::task::JoinSet;
 
+use crate::Db;
 use eyeballs_api::api_model;
 use eyeballs_common::grit;
 
-fn schedule_translations(
+type DbPool = <Db as Database>::Pool;
+type DbConnection = <DbPool as Pool>::Connection;
+
+fn schedule_translations<F, R>(
     tasks: &mut JoinSet<anyhow::Result<grit::TranslationFile>>,
     known: &mut HashSet<String>,
-    path: &Path,
+    opener: &F,
     files: &Vec<grit::IfFile>,
-) {
+) where
+    F: Fn(&str) -> anyhow::Result<BufReader<R>> + Clone + Send + 'static,
+    R: Read,
+{
     for file in files {
         match file {
             grit::IfFile::File(file) => {
                 if known.insert(file.path.to_string()) {
-                    tasks.spawn(grit::parse_xlf(path.join(file.path.as_str())));
+                    let path = file.path.to_string();
+                    let opener = opener.clone();
+                    tasks.spawn(async {
+                        grit::parse_xlf_with_opener(move || opener(path.as_str())).await
+                    });
                 }
             }
             grit::IfFile::If { expr: _, file } => {
-                schedule_translations(tasks, known, path, file);
+                schedule_translations(tasks, known, opener, file);
             }
         }
     }
@@ -29,6 +48,7 @@ fn schedule_translations(
 
 fn push_strings(
     strings: &mut Vec<api_model::LocalizationString>,
+    translation_ids: &mut Vec<i64>,
     file: &String,
     messages: Vec<grit::IfMessagePart>,
 ) {
@@ -71,15 +91,15 @@ fn push_strings(
                     source,
                     placeholders,
                     placeholder_offset,
-                    translation_id,
                     translations: Vec::<api_model::TranslationString>::new(),
                 });
+                translation_ids.push(translation_id);
             }
             grit::IfMessagePart::If { expr: _, message } => {
-                push_strings(strings, file, message);
+                push_strings(strings, translation_ids, file, message);
             }
             grit::IfMessagePart::Part { file, messages } => {
-                push_strings(strings, &file, messages);
+                push_strings(strings, translation_ids, &file, messages);
             }
         }
     }
@@ -126,52 +146,87 @@ fn push_translation(
         }
     }
 
-    string.translations.push(api_model::TranslationString {
-        language: language.to_string(),
-        translation,
-        placeholder_offset,
-    })
+    string.translations.sorted_insert_asc_by_key(
+        api_model::TranslationString {
+            language: language.to_string(),
+            translation,
+            placeholder_offset,
+            state: api_model::TranslationState::Unreviewed,
+            comment: "".to_string(),
+            reviewer: None,
+        },
+        |e| &e.language,
+    );
 }
 
 pub async fn collect_strings(
     base: impl AsRef<Path>,
     grits: impl IntoIterator<Item = String>,
 ) -> anyhow::Result<Vec<api_model::LocalizationString>> {
+    let base = base.as_ref().to_path_buf();
+    collect_strings_with_opener(grits, move |x| {
+        let path = base.join(x);
+        Ok(BufReader::new(File::open(path)?))
+    })
+    .await
+}
+
+pub async fn collect_strings_with_opener<F, R>(
+    grits: impl IntoIterator<Item = String>,
+    opener: F,
+) -> anyhow::Result<Vec<api_model::LocalizationString>>
+where
+    // TODO: Would like to avoid Sync here, it was possible in grit but not here
+    // for some reason.
+    F: Fn(&str) -> anyhow::Result<BufReader<R>> + Clone + Send + Sync + 'static,
+    R: Read,
+{
     let mut grit_tasks = JoinSet::new();
     for grit_name in grits {
-        let grit_path = base.as_ref().join(grit_name.as_str());
+        let opener_copy = opener.clone();
+        let grit_name_copy = grit_name.clone();
+        let grit_opener = move || opener_copy(grit_name_copy.as_str());
+
+        let part_opener = opener.clone();
         grit_tasks.spawn(async move {
-            let tmp = grit::parse_grit_with_parts(grit_path.as_path()).await;
-            (grit_path, grit_name, tmp)
+            let tmp = grit::parse_grit_with_parts_and_opener(grit_opener, part_opener).await;
+            (grit_name, tmp)
         });
     }
 
     let mut parsed_grits =
-        Vec::<(PathBuf, String, anyhow::Result<grit::Grit>)>::with_capacity(grit_tasks.len());
+        Vec::<(String, anyhow::Result<grit::Grit>)>::with_capacity(grit_tasks.len());
     while let Some(res) = grit_tasks.join_next().await {
         parsed_grits.push(res?);
     }
 
     let mut strings = Vec::<api_model::LocalizationString>::new();
+    let mut translation_ids = Vec::<i64>::new();
     let mut translation_tasks = JoinSet::new();
     let mut known_translations = HashSet::<String>::new();
 
-    for (grit_path, grit_name, maybe_grit) in parsed_grits {
+    for (grit_name, maybe_grit) in parsed_grits {
         let grit = maybe_grit?;
         schedule_translations(
             &mut translation_tasks,
             &mut known_translations,
-            grit_path.parent().unwrap(),
+            &opener,
             &grit.translations.file,
         );
 
         let first_index = strings.len();
 
-        push_strings(&mut strings, &grit_name, grit.release.messages.messages);
+        push_strings(
+            &mut strings,
+            &mut translation_ids,
+            &grit_name,
+            grit.release.messages.messages,
+        );
 
-        let mut id_to_string = HashMap::<i64, usize>::with_capacity(strings.len() - first_index);
-        for i in first_index..strings.len() {
-            id_to_string.insert(strings[i].translation_id, i);
+        let mut id_to_string =
+            HashMap::<i64, usize>::with_capacity(translation_ids.len() - first_index);
+        for i in first_index..translation_ids.len() {
+            id_to_string.insert(translation_ids[i], i);
         }
 
         while let Some(res) = translation_tasks.join_next().await {
@@ -190,3 +245,212 @@ pub async fn collect_strings(
 
     Ok(strings)
 }
+
+#[derive(Hash, PartialEq, Eq)]
+struct LocalizationStringKey<'a> {
+    file: Cow<'a, str>,
+    name: Cow<'a, str>,
+    meaning: Cow<'a, str>,
+}
+
+struct TranslationString {
+    base_translation: Option<String>,
+    base_placeholder_offsets: Option<String>,
+    head_translation: String,
+    head_placeholder_offsets: String,
+    state: api_model::TranslationState,
+}
+
+pub async fn review_add_strings(
+    db: &Db,
+    translation_reviewid: u64,
+    strings: Vec<api_model::LocalizationString>,
+    base: bool,
+) -> anyhow::Result<()> {
+    {
+        let mut tx = db.begin().await?;
+
+        let existing = sqlx::query!(
+            "SELECT id, name, file, meaning FROM localization_strings WHERE translation_review=?",
+            translation_reviewid
+        )
+        .fetch(&mut *tx)
+        .try_fold(HashMap::new(), async move |mut ex, r| {
+            ex.insert(
+                LocalizationStringKey {
+                    file: r.file.into(),
+                    name: r.name.into(),
+                    meaning: r.meaning.into(),
+                },
+                r.id,
+            );
+            Ok(ex)
+        })
+        .await
+        .unwrap();
+
+        for string in strings {
+            let key = LocalizationStringKey {
+                file: string.file.as_str().into(),
+                name: string.id.as_str().into(),
+                meaning: string.meaning.as_str().into(),
+            };
+            let id: u64;
+            let placeholder_offsets = string
+                .placeholder_offset
+                .into_iter()
+                .map(|x| x.to_string())
+                .collect::<Vec<String>>()
+                .join(",");
+            if let Some(existing_id) = existing.get(&key) {
+                sqlx::query!(
+                    "UPDATE localization_strings SET description=?, source=?, placeholder_offsets=? WHERE id=?",
+                    string.description,
+                    string.source,
+                    placeholder_offsets,
+                    existing_id)
+                    .execute(&mut *tx)
+                    .await
+                    .unwrap();
+                // TODO: Might be worth checking what needs updating but meh.
+                sqlx::query!(
+                    "DELETE FROM localization_placeholders WHERE localization_string=?",
+                    existing_id
+                )
+                .execute(&mut *tx)
+                .await
+                .unwrap();
+                id = *existing_id;
+            } else {
+                let result = sqlx::query!(
+                    "INSERT INTO localization_strings (translation_review, name, file, description, meaning, source, placeholder_offsets) VALUES (?, ?, ?, ?, ?, ?, ?)",
+                    translation_reviewid,
+                    string.id,
+                    string.file,
+                    string.description,
+                    string.meaning,
+                    string.source,
+                    placeholder_offsets)
+                    .execute(&mut* tx)
+                    .await
+                    .unwrap();
+                id = result.last_insert_id();
+            }
+
+            for placeholder in string.placeholders {
+                sqlx::query!(
+                    "INSERT INTO localization_placeholders (localization_string, name, content, example) VALUES (?, ?, ?, ?)",
+                    id,
+                    placeholder.id,
+                    placeholder.content,
+                    placeholder.example)
+                    .execute(&mut* tx)
+                    .await
+                    .unwrap();
+            }
+
+            if base {
+                sqlx::query!(
+                    "DELETE FROM translation_strings WHERE localization_string=?",
+                    id
+                )
+                .execute(&mut *tx)
+                .await
+                .unwrap();
+
+                for translation in string.translations {
+                    let placeholder_offsets = translation
+                        .placeholder_offset
+                        .into_iter()
+                        .map(|x| x.to_string())
+                        .collect::<Vec<String>>()
+                        .join(",");
+                    // Mark all as Unchanged as base == head here.
+                    sqlx::query!(
+                        "INSERT INTO translation_strings (localization_string, language, base_translation, base_placeholder_offsets, head_translation, head_placeholder_offsets, state) VALUES (?, ?, ?, ?, ?, ?, ?)",
+                        id,
+                        translation.language,
+                        translation.translation,
+                        placeholder_offsets,
+                        translation.translation,
+                        placeholder_offsets,
+                        u8::from(api_model::TranslationState::Unchanged))
+                        .execute(&mut* tx)
+                        .await
+                        .unwrap();
+                }
+            } else {
+                let existing = sqlx::query!("SELECT language, base_translation, base_placeholder_offsets, head_translation, head_placeholder_offsets, state FROM translation_strings WHERE localization_string=?", id)
+                    .fetch(&mut *tx)
+                    .try_fold(HashMap::new(), async move |mut ex, r| {
+                        ex.insert(r.language,
+                                  TranslationString {
+                                      base_translation: r.base_translation,
+                                      base_placeholder_offsets: r.base_placeholder_offsets,
+                                      head_translation: r.head_translation,
+                                      head_placeholder_offsets: r.head_placeholder_offsets,
+                                      state: api_model::TranslationState::try_from(r.state).unwrap_or(api_model::TranslationState::Unreviewed),
+                                  });
+                        Ok(ex)
+                    })
+                    .await
+                    .unwrap();
+
+                for translation in string.translations {
+                    let placeholder_offsets = translation
+                        .placeholder_offset
+                        .into_iter()
+                        .map(|x| x.to_string())
+                        .collect::<Vec<String>>()
+                        .join(",");
+
+                    if let Some(existing_translation) = existing.get(translation.language.as_str())
+                    {
+                        if existing_translation.head_translation != translation.translation
+                            || existing_translation.head_placeholder_offsets != placeholder_offsets
+                        {
+                            // Reset state whenever translation changes
+                            let new_state = if existing_translation
+                                .base_translation
+                                .as_ref()
+                                .is_some_and(|x| *x == translation.translation)
+                                && existing_translation
+                                    .base_placeholder_offsets
+                                    .as_ref()
+                                    .is_some_and(|x| *x == placeholder_offsets)
+                            {
+                                api_model::TranslationState::Unchanged
+                            } else {
+                                api_model::TranslationState::Unreviewed
+                            };
+                            sqlx::query!(
+                                "UPDATE translation_strings SET head_translation=?, head_placeholder_offsets=?, state=? WHERE localization_string = ? AND language = ?",
+                                translation.translation,
+                                placeholder_offsets,
+                                u8::from(new_state),
+                                id,
+                                translation.language)
+                                .execute(&mut* tx)
+                                .await
+                                .unwrap();
+                        }
+                    } else {
+                        sqlx::query!(
+                            "INSERT INTO translation_strings (localization_string, language, head_translation, head_placeholder_offsets) VALUES (?, ?, ?, ?)",
+                            id,
+                            translation.language,
+                            translation.translation,
+                            placeholder_offsets)
+                            .execute(&mut* tx)
+                            .await
+                            .unwrap();
+                    }
+                }
+            }
+        }
+
+        tx.commit().await?;
+    }
+
+    Ok(())
+}