From 2b54f5c51ff9a26d4077037631ed39d62ed2b3fb Mon Sep 17 00:00:00 2001 From: Joel Klinghed Date: Thu, 12 Jun 2025 09:11:18 +0200 Subject: Initial support for translation reviews --- server/src/trans.rs | 312 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 288 insertions(+), 24 deletions(-) (limited to 'server/src/trans.rs') diff --git a/server/src/trans.rs b/server/src/trans.rs index 6a18e27..c4e3b45 100644 --- a/server/src/trans.rs +++ b/server/src/trans.rs @@ -1,27 +1,46 @@ +#![allow(dead_code)] + use anyhow; +use futures::stream::TryStreamExt; +use rocket_db_pools::{sqlx, Database, Pool}; +use sorted_insert::SortedInsertByKey; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufReader, Read}; use std::iter::{repeat, IntoIterator}; -use std::path::{Path, PathBuf}; +use std::path::Path; use tokio::task::JoinSet; +use crate::Db; use eyeballs_api::api_model; use eyeballs_common::grit; -fn schedule_translations( +type DbPool = ::Pool; +type DbConnection = ::Connection; + +fn schedule_translations( tasks: &mut JoinSet>, known: &mut HashSet, - path: &Path, + opener: &F, files: &Vec, -) { +) where + F: Fn(&str) -> anyhow::Result> + Clone + Send + 'static, + R: Read, +{ for file in files { match file { grit::IfFile::File(file) => { if known.insert(file.path.to_string()) { - tasks.spawn(grit::parse_xlf(path.join(file.path.as_str()))); + let path = file.path.to_string(); + let opener = opener.clone(); + tasks.spawn(async { + grit::parse_xlf_with_opener(move || opener(path.as_str())).await + }); } } grit::IfFile::If { expr: _, file } => { - schedule_translations(tasks, known, path, file); + schedule_translations(tasks, known, opener, file); } } } @@ -29,6 +48,7 @@ fn schedule_translations( fn push_strings( strings: &mut Vec, + translation_ids: &mut Vec, file: &String, messages: Vec, ) { @@ -71,15 +91,15 @@ fn push_strings( source, placeholders, placeholder_offset, - translation_id, translations: Vec::::new(), }); + translation_ids.push(translation_id); } grit::IfMessagePart::If { expr: _, message } => { - push_strings(strings, file, message); + push_strings(strings, translation_ids, file, message); } grit::IfMessagePart::Part { file, messages } => { - push_strings(strings, &file, messages); + push_strings(strings, translation_ids, &file, messages); } } } @@ -126,52 +146,87 @@ fn push_translation( } } - string.translations.push(api_model::TranslationString { - language: language.to_string(), - translation, - placeholder_offset, - }) + string.translations.sorted_insert_asc_by_key( + api_model::TranslationString { + language: language.to_string(), + translation, + placeholder_offset, + state: api_model::TranslationState::Unreviewed, + comment: "".to_string(), + reviewer: None, + }, + |e| &e.language, + ); } pub async fn collect_strings( base: impl AsRef, grits: impl IntoIterator, ) -> anyhow::Result> { + let base = base.as_ref().to_path_buf(); + collect_strings_with_opener(grits, move |x| { + let path = base.join(x); + Ok(BufReader::new(File::open(path)?)) + }) + .await +} + +pub async fn collect_strings_with_opener( + grits: impl IntoIterator, + opener: F, +) -> anyhow::Result> +where + // TODO: Would like to avoid Sync here, it was possible in grit but not here + // for some reason. + F: Fn(&str) -> anyhow::Result> + Clone + Send + Sync + 'static, + R: Read, +{ let mut grit_tasks = JoinSet::new(); for grit_name in grits { - let grit_path = base.as_ref().join(grit_name.as_str()); + let opener_copy = opener.clone(); + let grit_name_copy = grit_name.clone(); + let grit_opener = move || opener_copy(grit_name_copy.as_str()); + + let part_opener = opener.clone(); grit_tasks.spawn(async move { - let tmp = grit::parse_grit_with_parts(grit_path.as_path()).await; - (grit_path, grit_name, tmp) + let tmp = grit::parse_grit_with_parts_and_opener(grit_opener, part_opener).await; + (grit_name, tmp) }); } let mut parsed_grits = - Vec::<(PathBuf, String, anyhow::Result)>::with_capacity(grit_tasks.len()); + Vec::<(String, anyhow::Result)>::with_capacity(grit_tasks.len()); while let Some(res) = grit_tasks.join_next().await { parsed_grits.push(res?); } let mut strings = Vec::::new(); + let mut translation_ids = Vec::::new(); let mut translation_tasks = JoinSet::new(); let mut known_translations = HashSet::::new(); - for (grit_path, grit_name, maybe_grit) in parsed_grits { + for (grit_name, maybe_grit) in parsed_grits { let grit = maybe_grit?; schedule_translations( &mut translation_tasks, &mut known_translations, - grit_path.parent().unwrap(), + &opener, &grit.translations.file, ); let first_index = strings.len(); - push_strings(&mut strings, &grit_name, grit.release.messages.messages); + push_strings( + &mut strings, + &mut translation_ids, + &grit_name, + grit.release.messages.messages, + ); - let mut id_to_string = HashMap::::with_capacity(strings.len() - first_index); - for i in first_index..strings.len() { - id_to_string.insert(strings[i].translation_id, i); + let mut id_to_string = + HashMap::::with_capacity(translation_ids.len() - first_index); + for i in first_index..translation_ids.len() { + id_to_string.insert(translation_ids[i], i); } while let Some(res) = translation_tasks.join_next().await { @@ -190,3 +245,212 @@ pub async fn collect_strings( Ok(strings) } + +#[derive(Hash, PartialEq, Eq)] +struct LocalizationStringKey<'a> { + file: Cow<'a, str>, + name: Cow<'a, str>, + meaning: Cow<'a, str>, +} + +struct TranslationString { + base_translation: Option, + base_placeholder_offsets: Option, + head_translation: String, + head_placeholder_offsets: String, + state: api_model::TranslationState, +} + +pub async fn review_add_strings( + db: &Db, + translation_reviewid: u64, + strings: Vec, + base: bool, +) -> anyhow::Result<()> { + { + let mut tx = db.begin().await?; + + let existing = sqlx::query!( + "SELECT id, name, file, meaning FROM localization_strings WHERE translation_review=?", + translation_reviewid + ) + .fetch(&mut *tx) + .try_fold(HashMap::new(), async move |mut ex, r| { + ex.insert( + LocalizationStringKey { + file: r.file.into(), + name: r.name.into(), + meaning: r.meaning.into(), + }, + r.id, + ); + Ok(ex) + }) + .await + .unwrap(); + + for string in strings { + let key = LocalizationStringKey { + file: string.file.as_str().into(), + name: string.id.as_str().into(), + meaning: string.meaning.as_str().into(), + }; + let id: u64; + let placeholder_offsets = string + .placeholder_offset + .into_iter() + .map(|x| x.to_string()) + .collect::>() + .join(","); + if let Some(existing_id) = existing.get(&key) { + sqlx::query!( + "UPDATE localization_strings SET description=?, source=?, placeholder_offsets=? WHERE id=?", + string.description, + string.source, + placeholder_offsets, + existing_id) + .execute(&mut *tx) + .await + .unwrap(); + // TODO: Might be worth checking what needs updating but meh. + sqlx::query!( + "DELETE FROM localization_placeholders WHERE localization_string=?", + existing_id + ) + .execute(&mut *tx) + .await + .unwrap(); + id = *existing_id; + } else { + let result = sqlx::query!( + "INSERT INTO localization_strings (translation_review, name, file, description, meaning, source, placeholder_offsets) VALUES (?, ?, ?, ?, ?, ?, ?)", + translation_reviewid, + string.id, + string.file, + string.description, + string.meaning, + string.source, + placeholder_offsets) + .execute(&mut* tx) + .await + .unwrap(); + id = result.last_insert_id(); + } + + for placeholder in string.placeholders { + sqlx::query!( + "INSERT INTO localization_placeholders (localization_string, name, content, example) VALUES (?, ?, ?, ?)", + id, + placeholder.id, + placeholder.content, + placeholder.example) + .execute(&mut* tx) + .await + .unwrap(); + } + + if base { + sqlx::query!( + "DELETE FROM translation_strings WHERE localization_string=?", + id + ) + .execute(&mut *tx) + .await + .unwrap(); + + for translation in string.translations { + let placeholder_offsets = translation + .placeholder_offset + .into_iter() + .map(|x| x.to_string()) + .collect::>() + .join(","); + // Mark all as Unchanged as base == head here. + sqlx::query!( + "INSERT INTO translation_strings (localization_string, language, base_translation, base_placeholder_offsets, head_translation, head_placeholder_offsets, state) VALUES (?, ?, ?, ?, ?, ?, ?)", + id, + translation.language, + translation.translation, + placeholder_offsets, + translation.translation, + placeholder_offsets, + u8::from(api_model::TranslationState::Unchanged)) + .execute(&mut* tx) + .await + .unwrap(); + } + } else { + let existing = sqlx::query!("SELECT language, base_translation, base_placeholder_offsets, head_translation, head_placeholder_offsets, state FROM translation_strings WHERE localization_string=?", id) + .fetch(&mut *tx) + .try_fold(HashMap::new(), async move |mut ex, r| { + ex.insert(r.language, + TranslationString { + base_translation: r.base_translation, + base_placeholder_offsets: r.base_placeholder_offsets, + head_translation: r.head_translation, + head_placeholder_offsets: r.head_placeholder_offsets, + state: api_model::TranslationState::try_from(r.state).unwrap_or(api_model::TranslationState::Unreviewed), + }); + Ok(ex) + }) + .await + .unwrap(); + + for translation in string.translations { + let placeholder_offsets = translation + .placeholder_offset + .into_iter() + .map(|x| x.to_string()) + .collect::>() + .join(","); + + if let Some(existing_translation) = existing.get(translation.language.as_str()) + { + if existing_translation.head_translation != translation.translation + || existing_translation.head_placeholder_offsets != placeholder_offsets + { + // Reset state whenever translation changes + let new_state = if existing_translation + .base_translation + .as_ref() + .is_some_and(|x| *x == translation.translation) + && existing_translation + .base_placeholder_offsets + .as_ref() + .is_some_and(|x| *x == placeholder_offsets) + { + api_model::TranslationState::Unchanged + } else { + api_model::TranslationState::Unreviewed + }; + sqlx::query!( + "UPDATE translation_strings SET head_translation=?, head_placeholder_offsets=?, state=? WHERE localization_string = ? AND language = ?", + translation.translation, + placeholder_offsets, + u8::from(new_state), + id, + translation.language) + .execute(&mut* tx) + .await + .unwrap(); + } + } else { + sqlx::query!( + "INSERT INTO translation_strings (localization_string, language, head_translation, head_placeholder_offsets) VALUES (?, ?, ?, ?)", + id, + translation.language, + translation.translation, + placeholder_offsets) + .execute(&mut* tx) + .await + .unwrap(); + } + } + } + } + + tx.commit().await?; + } + + Ok(()) +} -- cgit v1.2.3-70-g09d2