#![allow(dead_code)] use anyhow; use futures::stream::TryStreamExt; use rocket_db_pools::{sqlx, Database, Pool}; use sorted_insert::SortedInsertByKey; use sqlx::Acquire; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{BufReader, Read}; use std::iter::{repeat, IntoIterator}; use std::path::Path; use tokio::task::JoinSet; use crate::Db; use eyeballs_api::api_model; use eyeballs_common::grit; type DbPool = ::Pool; type DbConnection = ::Connection; fn schedule_translations( tasks: &mut JoinSet>, known: &mut HashSet, opener: &F, files: &Vec, ) where F: Fn(&str) -> anyhow::Result> + Clone + Send + 'static, R: Read, { for file in files { match file { grit::IfFile::File(file) => { if known.insert(file.path.to_string()) { let path = file.path.to_string(); let opener = opener.clone(); tasks.spawn(async { grit::parse_xlf_with_opener(move || opener(path.as_str())).await }); } } grit::IfFile::If { expr: _, file } => { schedule_translations(tasks, known, opener, file); } } } } fn push_strings( strings: &mut Vec, translation_ids: &mut Vec, file: &String, messages: Vec, ) { for message in messages { match message { grit::IfMessagePart::Message(message) => { let mut source = String::new(); let mut placeholders = Vec::::new(); let mut placeholder_offset = Vec::::new(); let translation_id = grit::get_message_id(&message); let mut offset: usize = 0; for text in message.content { match text { grit::TextPlaceholder::Text(text) => { source.push_str(text.as_str()); offset += text.len(); } grit::TextPlaceholder::Placeholder { name, content, example, } => { placeholders.push(api_model::LocalizationPlaceholder { id: name, content, example: example.unwrap_or_default(), }); placeholder_offset.push(offset); } } } strings.push(api_model::LocalizationString { id: message.name, file: file.to_string(), description: message.desc, meaning: message.meaning.unwrap_or_default(), source, placeholders, placeholder_offset, translations: Vec::::new(), }); translation_ids.push(translation_id); } grit::IfMessagePart::If { expr: _, message } => { push_strings(strings, translation_ids, file, message); } grit::IfMessagePart::Part { file, messages } => { push_strings(strings, translation_ids, &file, messages); } } } } fn push_translation( string: &mut api_model::LocalizationString, language: &String, unit: grit::TranslationUnit, ) { let mut translation = String::new(); let mut placeholder_offset = Vec::::with_capacity(string.placeholders.len()); // Fill offset vec with zeros, it's not guaranteed that they will be in the same order // below so easier to index directly. placeholder_offset.extend(repeat(0).take(string.placeholders.len())); // There can be multiple placeholders with the same name, so when doing name lookup, // skip the previous hits. let mut placeholder_last = HashMap::::new(); let mut offset: usize = 0; for text in unit.target { match text { grit::TextPlaceholder::Text(text) => { translation.push_str(text.as_str()); offset += text.len(); } grit::TextPlaceholder::Placeholder { name, content: _, example: _, } => { let previous = placeholder_last.get(name.as_str()).map_or(0, |x| x + 1); if let Some(index) = string .placeholders .iter() .skip(previous) .position(|x| x.id == name) { placeholder_last.insert(name, previous + index); placeholder_offset[previous + index] = offset; } } } } string.translations.sorted_insert_asc_by_key( api_model::TranslationString { language: language.to_string(), translation, placeholder_offset, state: api_model::TranslationState::Unreviewed, comment: "".to_string(), reviewer: None, }, |e| &e.language, ); } pub async fn collect_strings( base: impl AsRef, grits: impl IntoIterator, ) -> anyhow::Result> { let base = base.as_ref().to_path_buf(); collect_strings_with_opener(grits, move |x| { let path = base.join(x); Ok(BufReader::new(File::open(path)?)) }) .await } pub async fn collect_strings_with_opener( grits: impl IntoIterator, opener: F, ) -> anyhow::Result> where // TODO: Would like to avoid Sync here, it was possible in grit but not here // for some reason. F: Fn(&str) -> anyhow::Result> + Clone + Send + Sync + 'static, R: Read, { let mut grit_tasks = JoinSet::new(); for grit_name in grits { let opener_copy = opener.clone(); let grit_name_copy = grit_name.clone(); let grit_opener = move || opener_copy(grit_name_copy.as_str()); let part_opener = opener.clone(); grit_tasks.spawn(async move { let tmp = grit::parse_grit_with_parts_and_opener(grit_opener, part_opener).await; (grit_name, tmp) }); } let mut parsed_grits = Vec::<(String, anyhow::Result)>::with_capacity(grit_tasks.len()); while let Some(res) = grit_tasks.join_next().await { parsed_grits.push(res?); } let mut strings = Vec::::new(); let mut translation_ids = Vec::::new(); let mut translation_tasks = JoinSet::new(); let mut known_translations = HashSet::::new(); for (grit_name, maybe_grit) in parsed_grits { let grit = maybe_grit?; schedule_translations( &mut translation_tasks, &mut known_translations, &opener, &grit.translations.file, ); let first_index = strings.len(); push_strings( &mut strings, &mut translation_ids, &grit_name, grit.release.messages.messages, ); let mut id_to_string = HashMap::::with_capacity(translation_ids.len() - first_index); for (i, id) in translation_ids.iter().enumerate().skip(first_index) { id_to_string.insert(*id, i); } while let Some(res) = translation_tasks.join_next().await { let translation_file = res??; for unit in translation_file.units { if let Some(index) = id_to_string.get(&unit.id) { push_translation( &mut strings[*index], &translation_file.target_language, unit, ); } } } } Ok(strings) } #[derive(Hash, PartialEq, Eq)] struct LocalizationStringKey<'a> { file: Cow<'a, str>, name: Cow<'a, str>, meaning: Cow<'a, str>, } struct TranslationString { base_translation: Option, base_placeholder_offsets: Option, head_translation: String, head_placeholder_offsets: String, state: api_model::TranslationState, } pub async fn review_add_strings( db: &mut DbConnection, translation_reviewid: u64, strings: Vec, base: bool, ) -> anyhow::Result<()> { { let mut tx = db.begin().await?; let existing = sqlx::query!( "SELECT id, name, file, meaning FROM localization_strings WHERE translation_review=?", translation_reviewid ) .fetch(&mut *tx) .try_fold(HashMap::new(), async move |mut ex, r| { ex.insert( LocalizationStringKey { file: r.file.into(), name: r.name.into(), meaning: r.meaning.into(), }, r.id, ); Ok(ex) }) .await .unwrap(); for string in strings { let key = LocalizationStringKey { file: string.file.as_str().into(), name: string.id.as_str().into(), meaning: string.meaning.as_str().into(), }; let id: u64; let placeholder_offsets = string .placeholder_offset .into_iter() .map(|x| x.to_string()) .collect::>() .join(","); if let Some(existing_id) = existing.get(&key) { sqlx::query!( "UPDATE localization_strings SET description=?, source=?, placeholder_offsets=? WHERE id=?", string.description, string.source, placeholder_offsets, existing_id) .execute(&mut *tx) .await .unwrap(); // TODO: Might be worth checking what needs updating but meh. sqlx::query!( "DELETE FROM localization_placeholders WHERE localization_string=?", existing_id ) .execute(&mut *tx) .await .unwrap(); id = *existing_id; } else { let result = sqlx::query!( "INSERT INTO localization_strings (translation_review, name, file, description, meaning, source, placeholder_offsets) VALUES (?, ?, ?, ?, ?, ?, ?)", translation_reviewid, string.id, string.file, string.description, string.meaning, string.source, placeholder_offsets) .execute(&mut* tx) .await .unwrap(); id = result.last_insert_id(); } for placeholder in string.placeholders { sqlx::query!( "INSERT INTO localization_placeholders (localization_string, name, content, example) VALUES (?, ?, ?, ?)", id, placeholder.id, placeholder.content, placeholder.example) .execute(&mut* tx) .await .unwrap(); } if base { sqlx::query!( "DELETE FROM translation_strings WHERE localization_string=?", id ) .execute(&mut *tx) .await .unwrap(); for translation in string.translations { let placeholder_offsets = translation .placeholder_offset .into_iter() .map(|x| x.to_string()) .collect::>() .join(","); // Mark all as Unchanged as base == head here. sqlx::query!( "INSERT INTO translation_strings (localization_string, language, base_translation, base_placeholder_offsets, head_translation, head_placeholder_offsets, state) VALUES (?, ?, ?, ?, ?, ?, ?)", id, translation.language, translation.translation, placeholder_offsets, translation.translation, placeholder_offsets, u8::from(api_model::TranslationState::Unchanged)) .execute(&mut* tx) .await .unwrap(); } } else { let existing = sqlx::query!("SELECT language, base_translation, base_placeholder_offsets, head_translation, head_placeholder_offsets, state FROM translation_strings WHERE localization_string=?", id) .fetch(&mut *tx) .try_fold(HashMap::new(), async move |mut ex, r| { ex.insert(r.language, TranslationString { base_translation: r.base_translation, base_placeholder_offsets: r.base_placeholder_offsets, head_translation: r.head_translation, head_placeholder_offsets: r.head_placeholder_offsets, state: api_model::TranslationState::try_from(r.state).unwrap_or(api_model::TranslationState::Unreviewed), }); Ok(ex) }) .await .unwrap(); for translation in string.translations { let placeholder_offsets = translation .placeholder_offset .into_iter() .map(|x| x.to_string()) .collect::>() .join(","); if let Some(existing_translation) = existing.get(translation.language.as_str()) { if existing_translation.head_translation != translation.translation || existing_translation.head_placeholder_offsets != placeholder_offsets { // Reset state whenever translation changes let new_state = if existing_translation .base_translation .as_ref() .is_some_and(|x| *x == translation.translation) && existing_translation .base_placeholder_offsets .as_ref() .is_some_and(|x| *x == placeholder_offsets) { api_model::TranslationState::Unchanged } else { api_model::TranslationState::Unreviewed }; sqlx::query!( "UPDATE translation_strings SET head_translation=?, head_placeholder_offsets=?, state=? WHERE localization_string = ? AND language = ?", translation.translation, placeholder_offsets, u8::from(new_state), id, translation.language) .execute(&mut* tx) .await .unwrap(); } } else { sqlx::query!( "INSERT INTO translation_strings (localization_string, language, head_translation, head_placeholder_offsets) VALUES (?, ?, ?, ?)", id, translation.language, translation.translation, placeholder_offsets) .execute(&mut* tx) .await .unwrap(); } } } } tx.commit().await?; } Ok(()) }