diff options
Diffstat (limited to 'server/src')
| -rw-r--r-- | server/src/main.rs | 1 | ||||
| -rw-r--r-- | server/src/tests.rs | 197 | ||||
| -rw-r--r-- | server/src/trans.rs | 193 |
3 files changed, 391 insertions, 0 deletions
diff --git a/server/src/main.rs b/server/src/main.rs index 973febe..66faec3 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -26,6 +26,7 @@ mod auth; mod authorized_keys; mod db_utils; mod git_root; +mod trans; use auth::AuthApiAddon; diff --git a/server/src/tests.rs b/server/src/tests.rs index cc71bf3..5f956fb 100644 --- a/server/src/tests.rs +++ b/server/src/tests.rs @@ -1,3 +1,4 @@ +use pretty_assertions::assert_eq; use rocket::figment::util::map; use rocket::figment::value::{Map, Value}; use rocket::http::{ContentType, Header, Status}; @@ -5,11 +6,13 @@ use rocket::local::asynchronous::{Client, LocalRequest}; use sqlx::mysql::{MySql, MySqlConnectOptions, MySqlPoolOptions}; use sqlx::{Acquire, Executor, Pool}; use std::fmt::Display; +use std::path::PathBuf; use std::sync::OnceLock; use stdext::function_name; use testdir::testdir; use crate::api_model; +use crate::trans; struct RealIP(&'static str); @@ -966,3 +969,197 @@ async fn test_user_keys_del() { assert_eq!(user_keys.more, false); assert_eq!(user_keys.keys.len(), 0); } + +#[tokio::test] +async fn test_collect_strings() { + let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("common/src/testdata/grit"); + let mut strings = trans::collect_strings(base, vec!["base.grd".to_string()]) + .await + .unwrap(); + + // Sort translations before comparison, they come unsorted from collect_strings. + for string in &mut strings { + string + .translations + .sort_by(|a, b| a.language.cmp(&b.language)); + } + + assert_eq!( + strings, + vec![ + api_model::LocalizationString { + id: "IDS_EXTRA".to_string(), + // TODO: Should be extra.grpd + file: "base.grd".to_string(), + description: "Extra title".to_string(), + meaning: "".to_string(), + source: "Extra title".to_string(), + placeholders: vec![], + placeholder_offset: vec![], + translation_id: 3567801202192813433, + translations: vec![], + }, + api_model::LocalizationString { + id: "IDS_BOOKMARKS_FRAGMENT_TITLE".to_string(), + file: "base.grd".to_string(), + description: "Title which is shown on the main bookmarks view.".to_string(), + meaning: "".to_string(), + source: "Bookmarks".to_string(), + placeholders: vec![], + placeholder_offset: vec![], + translation_id: 8820817407110198400, + translations: vec![ + api_model::TranslationString { + language: "en-gb".to_string(), + translation: "Bookmarks".to_string(), + placeholder_offset: vec![], + }, + api_model::TranslationString { + language: "my".to_string(), + translation: "ဝက်ဘ်လိပ်စာ မှတ်ထားမှုများ".to_string(), + placeholder_offset: vec![], + }, + api_model::TranslationString { + language: "my-zg".to_string(), + translation: "ဝက္ဘ္လိပ္စာ မွတ္ထားမွုမ်ား".to_string(), + placeholder_offset: vec![], + }, + api_model::TranslationString { + language: "sv".to_string(), + translation: "Bokmärken".to_string(), + placeholder_offset: vec![], + }, + ], + }, + api_model::LocalizationString { + id: "IDS_GENERIC_WELCOME".to_string(), + file: "base.grd".to_string(), + description: "Generic welcome string.".to_string(), + meaning: "".to_string(), + source: "Welcome to ".to_string(), + placeholders: vec![ + api_model::LocalizationPlaceholder { + id: "STRING".to_string(), + content: "%1$s".to_string(), + example: "Opera".to_string(), + }, + ], + placeholder_offset: vec![11], + translation_id: 8443102241046796905, + translations: vec![ + api_model::TranslationString { + language: "en-gb".to_string(), + translation: "Welcome to ".to_string(), + placeholder_offset: vec![11], + }, + api_model::TranslationString { + language: "my".to_string(), + translation: " မှ ကြိုဆိုပါသည်".to_string(), + placeholder_offset: vec![0], + }, + api_model::TranslationString { + language: "my-zg".to_string(), + translation: " မွ ႀကိဳဆိုပါသည္".to_string(), + placeholder_offset: vec![0], + }, + api_model::TranslationString { + language: "sv".to_string(), + translation: "Välkommen till ".to_string(), + placeholder_offset: vec![16], + }, + ], + }, + api_model::LocalizationString { + id: "IDS_START_TERMS".to_string(), + file: "base.grd".to_string(), + description: "First startup information about the license and privacy terms.".to_string(), + meaning: "".to_string(), + source: "By using this application you are agreeing to Opera's Terms of Service. Also, you can learn how Opera handles and protects your data in our Privacy Statement.".to_string(), + placeholders: vec![ + api_model::LocalizationPlaceholder { + id: "TOS_BEGIN".to_string(), + content: "<tos>".to_string(), + example: "".to_string(), + }, + api_model::LocalizationPlaceholder { + id: "TOS_END".to_string(), + content: "</tos>".to_string(), + example: "".to_string(), + }, + api_model::LocalizationPlaceholder { + id: "PRIVACY_BEGIN".to_string(), + content: "<privacy>".to_string(), + example: "".to_string(), + }, + api_model::LocalizationPlaceholder { + id: "PRIVACY_END".to_string(), + content: "</privacy>".to_string(), + example: "".to_string(), + }, + ], + placeholder_offset: vec![54, 70, 140, 157], + translation_id: 2466140279568640908, + translations: vec![ + api_model::TranslationString { + language: "en-gb".to_string(), + translation: "By using this application you are agreeing to Opera's Terms of Service. Also, you can learn how Opera handles and protects your data in our Privacy Statement.".to_string(), + placeholder_offset: vec![54, 70, 140, 157], + }, + api_model::TranslationString { + language: "my".to_string(), + translation: "ဤအပလီကေးရှင်းကို အသုံးပြုခြင်းဖြင့် သင်သည် Opera ၏ ဝန်ဆောင်မှုစည်းမျဉ်းများ ကို သဘောတူရာ ရောက်ပါသည်။ ထို့အပြင် ကျွန်ုပ်တို့၏ကိုယ်ရေးလုံခြုံမှု ထုတ်ပြန်ချက် ထဲတွင် သင့်ဒေတာများကို Opera ၏ ကိုင်တွယ်ပုံနှင့် ကာကွယ်ပုံတို့ကိုလည်း လေ့လာနိုင်သည်။".to_string(), + placeholder_offset: vec![133, 205, 342, 433], + }, + api_model::TranslationString { + language: "my-zg".to_string(), + translation: "ဤအပလီေကးရွင္းကို အသုံးျပဳျခင္းျဖင့္ သင္သည္ Opera ၏ ဝန္ေဆာင္မွုစည္းမ်ဥ္းမ်ား ကို သေဘာတူရာ ေရာက္ပါသည္။ ထို႔အျပင္ ကၽြန္ုပ္တို႔၏ကိုယ္ေရးလုံျခဳံမွု ထုတ္ျပန္ခ်က္ ထဲတြင္ သင့္ေဒတာမ်ားကို Opera ၏ ကိုင္တြယ္ပုံႏွင့္ ကာကြယ္ပုံတို႔ကိုလည္း ေလ့လာနိုင္သည္။".to_string(), + placeholder_offset: vec![133, 205, 342, 433], + }, + api_model::TranslationString { + language: "sv".to_string(), + translation: "I och med din användning av det här programmet samtycker du till Operas Licensvillkor. Du kan också läsa om hur Opera hanterar och skyddar dina data i vårt Sekretessmeddelande.".to_string(), + placeholder_offset: vec![74, 87, 161, 180], + }, + ], + }, + api_model::LocalizationString { + id: "IDS_BOOKMARKS_FOLDERS_DELETED".to_string(), + file: "base.grd".to_string(), + description: "Message which is shown when one or more folders have been deleted from the bookmark list.".to_string(), + meaning: "".to_string(), + source: "{BOOKMARKS, plural,\n one { folder deleted}\n few { folders deleted}\n many { folders deleted}\n other { folders deleted}}".to_string(), + placeholders: vec![ + api_model::LocalizationPlaceholder { + id: "COUNT".to_string(), + content: "%1$d".to_string(), + example: "1".to_string(), + }, + api_model::LocalizationPlaceholder { + id: "COUNT".to_string(), + content: "%1$d".to_string(), + example: "15".to_string(), + }, + api_model::LocalizationPlaceholder { + id: "COUNT".to_string(), + content: "%1$d".to_string(), + example: "100".to_string(), + }, + api_model::LocalizationPlaceholder { + id: "COUNT".to_string(), + content: "%1$d".to_string(), + example: "42".to_string(), + }, + ], + placeholder_offset: vec![34, 65, 98, 132], + translation_id: 7770247413830876286, + translations: vec![ + api_model::TranslationString { + language: "en-gb".to_string(), + translation: "{BOOKMARKS, plural,\n one { folder deleted}\n few { folders deleted}\n many { folders deleted}\n other { folders deleted}}".to_string(), + placeholder_offset: vec![35, 67, 101, 136], + }, + ], + }, + ], + ) +} diff --git a/server/src/trans.rs b/server/src/trans.rs new file mode 100644 index 0000000..dcef078 --- /dev/null +++ b/server/src/trans.rs @@ -0,0 +1,193 @@ +use anyhow; +use std::collections::{HashMap, HashSet}; +use std::iter::{repeat, IntoIterator}; +use std::path::{Path, PathBuf}; +use tokio::task::JoinSet; + +use eyeballs_api::api_model; +use eyeballs_common::grit; + +fn schedule_translations( + tasks: &mut JoinSet<anyhow::Result<grit::TranslationFile>>, + known: &mut HashSet<String>, + path: &Path, + files: &Vec<grit::IfFile>, +) { + for file in files { + match file { + grit::IfFile::File(file) => { + if known.insert(file.path.to_string()) { + tasks.spawn(grit::parse_xlf(path.join(file.path.as_str()))); + } + } + grit::IfFile::If { expr: _, file } => { + schedule_translations(tasks, known, path, file); + } + } + } +} + +fn push_strings( + strings: &mut Vec<api_model::LocalizationString>, + file: &String, + messages: Vec<grit::IfMessagePart>, +) { + for message in messages { + match message { + grit::IfMessagePart::Message(message) => { + let mut source = String::new(); + let mut placeholders = Vec::<api_model::LocalizationPlaceholder>::new(); + let mut placeholder_offset = Vec::<usize>::new(); + + let translation_id = grit::get_message_id(&message); + + let mut offset: usize = 0; + for text in message.content { + match text { + grit::TextPlaceholder::Text(text) => { + source.push_str(text.as_str()); + offset += text.len(); + } + grit::TextPlaceholder::Placeholder { + name, + content, + example, + } => { + placeholders.push(api_model::LocalizationPlaceholder { + id: name, + content, + example: example.unwrap_or_default(), + }); + placeholder_offset.push(offset); + } + } + } + + strings.push(api_model::LocalizationString { + id: message.name, + file: file.to_string(), + description: message.desc, + meaning: message.meaning.unwrap_or_default(), + source, + placeholders, + placeholder_offset, + translation_id, + translations: Vec::<api_model::TranslationString>::new(), + }); + } + grit::IfMessagePart::If { expr: _, message } => { + push_strings(strings, file, message); + } + grit::IfMessagePart::Part(_) => { + // There should be none of these as we use parse_grit_with_parts + assert!(false); + } + } + } +} + +fn push_translation( + string: &mut api_model::LocalizationString, + language: &String, + unit: grit::TranslationUnit, +) { + let mut translation = String::new(); + let mut placeholder_offset = Vec::<usize>::with_capacity(string.placeholders.len()); + // Fill offset vec with zeros, it's not guaranteed that they will be in the same order + // below so easier to index directly. + placeholder_offset.extend(repeat(0).take(string.placeholders.len())); + + // There can be multiple placeholders with the same name, so when doing name lookup, + // skip the previous hits. + let mut placeholder_last = HashMap::<String, usize>::new(); + + let mut offset: usize = 0; + for text in unit.target { + match text { + grit::TextPlaceholder::Text(text) => { + translation.push_str(text.as_str()); + offset += text.len(); + } + grit::TextPlaceholder::Placeholder { + name, + content: _, + example: _, + } => { + let previous = placeholder_last.get(name.as_str()).map_or(0, |x| x + 1); + if let Some(index) = string + .placeholders + .iter() + .skip(previous) + .position(|x| x.id == name) + { + placeholder_last.insert(name, previous + index); + placeholder_offset[previous + index] = offset; + } + } + } + } + + string.translations.push(api_model::TranslationString { + language: language.to_string(), + translation, + placeholder_offset, + }) +} + +pub async fn collect_strings( + base: impl AsRef<Path>, + grits: impl IntoIterator<Item = String>, +) -> anyhow::Result<Vec<api_model::LocalizationString>> { + let mut grit_tasks = JoinSet::new(); + for grit_name in grits { + let grit_path = base.as_ref().join(grit_name.as_str()); + grit_tasks.spawn(async move { + let tmp = grit::parse_grit_with_parts(grit_path.as_path()).await; + (grit_path, grit_name, tmp) + }); + } + + let mut parsed_grits = + Vec::<(PathBuf, String, anyhow::Result<grit::Grit>)>::with_capacity(grit_tasks.len()); + while let Some(res) = grit_tasks.join_next().await { + parsed_grits.push(res?); + } + + let mut strings = Vec::<api_model::LocalizationString>::new(); + let mut translation_tasks = JoinSet::new(); + let mut known_translations = HashSet::<String>::new(); + + for (grit_path, grit_name, maybe_grit) in parsed_grits { + let grit = maybe_grit?; + schedule_translations( + &mut translation_tasks, + &mut known_translations, + grit_path.parent().unwrap(), + &grit.translations.file, + ); + + let first_index = strings.len(); + + push_strings(&mut strings, &grit_name, grit.release.messages.messages); + + let mut id_to_string = HashMap::<i64, usize>::with_capacity(strings.len() - first_index); + for i in first_index..strings.len() { + id_to_string.insert(strings[i].translation_id, i); + } + + while let Some(res) = translation_tasks.join_next().await { + let translation_file = res??; + for unit in translation_file.units { + if let Some(index) = id_to_string.get(&unit.id) { + push_translation( + &mut strings[*index], + &translation_file.target_language, + unit, + ); + } + } + } + } + + Ok(strings) +} |