#![allow(dead_code)] use anyhow::Error; use std::collections::VecDeque; use std::fs; use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use tokio::task::spawn_blocking; use xml::attribute::OwnedAttribute; use xml::reader::{EventReader, ParserConfig, XmlEvent}; #[derive(Debug, PartialEq)] pub struct Grit { pub current_release: u32, pub latest_public_release: u32, pub outputs: Outputs, pub translations: Translations, pub release: Release, } #[derive(Debug, PartialEq)] pub struct Outputs { pub output: Vec, } #[derive(Debug, PartialEq)] pub struct Output { pub filename: String, pub output_type: String, pub lang: String, } #[derive(Debug, PartialEq)] pub enum IfOutput { Output(Output), If { expr: String, output: Vec }, } #[derive(Debug, PartialEq)] pub struct Translations { pub file: Vec, } #[derive(Debug, PartialEq)] pub struct File { pub path: String, pub lang: String, } #[derive(Debug, PartialEq)] pub enum IfFile { File(File), If { expr: String, file: Vec }, } #[derive(Debug, PartialEq)] pub struct Release { pub allow_pseudo: bool, pub seq: u32, pub messages: Messages, } #[derive(Debug, PartialEq)] pub struct Messages { pub fallback_to_english: bool, pub messages: Vec, } #[derive(Debug, PartialEq)] pub enum IfMessagePart { Message(Message), Part { file: String, messages: Vec, }, If { expr: String, message: Vec, }, } #[derive(Debug, PartialEq)] pub struct Message { pub desc: String, pub name: String, pub internal_comment: Option, pub meaning: Option, pub content: Vec, } #[derive(Debug, PartialEq)] pub enum TextPlaceholder { Text(String), Placeholder { name: String, content: String, example: Option, }, } #[derive(Debug, PartialEq)] pub enum IfMessage { Message(Message), If { expr: String, message: Vec, }, } #[derive(Debug, PartialEq)] pub struct GritPart { pub messages: Vec, } #[derive(Debug, PartialEq)] pub struct TranslationFile { pub target_language: String, pub units: Vec, } #[derive(Debug, PartialEq)] pub struct TranslationUnit { pub id: i64, pub target: Vec, } fn get_opt_attribute<'a>(attributes: &'a [OwnedAttribute], name: &str) -> Option<&'a str> { for attribute in attributes { if attribute.name.local_name == name { return Some(attribute.value.as_str()); } } None } fn get_attribute<'a>(attributes: &'a [OwnedAttribute], name: &str) -> anyhow::Result<&'a str> { get_opt_attribute(attributes, name).ok_or(Error::msg(format!("Expected attribute {name}"))) } fn parse_grit_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let current_release = get_attribute(attributes, "current_release")?.parse::()?; let latest_public_release = get_attribute(attributes, "latest_public_release")?.parse::()?; let mut outputs: Option = None; let mut translations: Option = None; let mut release: Option = None; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "outputs" => { outputs = Some(parse_outputs_element(&attributes, reader)?); } "translations" => { translations = Some(parse_translations_element(&attributes, reader)?); } "release" => { release = Some(parse_release_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in grit", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "grit"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(Grit { current_release, latest_public_release, outputs: outputs.ok_or(Error::msg("Expected outputs in grit"))?, translations: translations.ok_or(Error::msg("Expected outputs in grit"))?, release: release.ok_or(Error::msg("Expected outputs in grit"))?, }) } fn parse_outputs_element( _attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let mut output = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "if" => { output.push(parse_if_output_element(&attributes, reader)?); } "output" => { output.push(IfOutput::Output(parse_output_element(&attributes, reader)?)); } _ => { return Err(Error::msg(format!( "Unexpected {0} in outputs", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "outputs"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(Outputs { output }) } fn parse_output_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let filename = get_attribute(attributes, "filename")?; let output_type = get_attribute(attributes, "type")?; let lang = get_attribute(attributes, "lang")?; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes: _, namespace: _, } => { return Err(Error::msg(format!( "Unexpected {0} in output", name.local_name ))); } XmlEvent::EndElement { name } => { assert!(name.local_name == "output"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(Output { filename: filename.to_string(), output_type: output_type.to_string(), lang: lang.to_string(), }) } fn parse_if_output_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let expr = get_attribute(attributes, "expr")?; let mut output = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "output" => { output.push(IfOutput::Output(parse_output_element(&attributes, reader)?)); } "if" => { output.push(parse_if_output_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in outputs>if", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "if"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(IfOutput::If { expr: expr.to_string(), output, }) } fn parse_translations_element( _attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let mut file = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "if" => { file.push(parse_if_file_element(&attributes, reader)?); } "file" => { file.push(IfFile::File(parse_file_element(&attributes, reader)?)); } _ => { return Err(Error::msg(format!( "Unexpected {0} in translations", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "translations"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(Translations { file }) } fn parse_file_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let path = get_attribute(attributes, "path")?; let lang = get_attribute(attributes, "lang")?; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes: _, namespace: _, } => { return Err(Error::msg(format!( "Unexpected {0} in file", name.local_name ))); } XmlEvent::EndElement { name } => { assert!(name.local_name == "file"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(File { path: path.to_string(), lang: lang.to_string(), }) } fn parse_if_file_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let expr = get_attribute(attributes, "expr")?; let mut file = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "file" => { file.push(IfFile::File(parse_file_element(&attributes, reader)?)); } "if" => { file.push(parse_if_file_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in outputs>if", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "if"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(IfFile::If { expr: expr.to_string(), file, }) } fn parse_release_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let allow_pseudo = get_attribute(attributes, "allow_pseudo")?.parse::()?; let seq = get_attribute(attributes, "seq")?.parse::()?; let mut messages: Option = None; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "messages" => { messages = Some(parse_messages_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in release", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "release"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(Release { allow_pseudo, seq, messages: messages.ok_or(Error::msg("No messages in release"))?, }) } fn parse_messages_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let fallback_to_english = get_attribute(attributes, "fallback_to_english")?.parse::()?; let mut messages = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "message" => { messages.push(IfMessagePart::Message(parse_message_element( &attributes, reader, )?)); } "part" => { messages.push(parse_part_element(&attributes, reader)?); } "if" => { messages.push(parse_if_message_part_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in messages", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "messages"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(Messages { fallback_to_english, messages, }) } fn parse_message_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let desc = get_attribute(attributes, "desc")?; let name = get_attribute(attributes, "name")?; let internal_comment = get_opt_attribute(attributes, "internal_comment").map(|s| s.to_string()); let meaning = get_opt_attribute(attributes, "meaning").map(|s| s.to_string()); let mut content = Vec::::new(); let mut first = true; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "ph" => { first = false; content.push(parse_placeholder_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in file", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "message"); break; } XmlEvent::Characters(data) => content.push(TextPlaceholder::Text(if first { first = false; data.trim_start().to_string() } else { data })), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } if !first { match content.last_mut().unwrap() { TextPlaceholder::Text(data) => { data.truncate(data.trim_end().len()); if data.is_empty() { content.pop(); } } TextPlaceholder::Placeholder { name: _, content: _, example: _, } => {} } } Ok(Message { desc: desc.to_string(), name: name.to_string(), internal_comment, meaning, content, }) } fn parse_if_message_part_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let expr = get_attribute(attributes, "expr")?; let mut message = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "message" => { message.push(IfMessagePart::Message(parse_message_element( &attributes, reader, )?)); } "part" => { message.push(parse_part_element(&attributes, reader)?); } "if" => { message.push(parse_if_message_part_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in messages>if", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "if"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(IfMessagePart::If { expr: expr.to_string(), message, }) } fn parse_if_message_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let expr = get_attribute(attributes, "expr")?; let mut message = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "message" => { message.push(IfMessage::Message(parse_message_element( &attributes, reader, )?)); } "if" => { message.push(parse_if_message_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in grit-part>if", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "if"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(IfMessage::If { expr: expr.to_string(), message, }) } fn parse_part_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let file = get_attribute(attributes, "file")?; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes: _, namespace: _, } => { return Err(Error::msg(format!( "Unexpected {0} in part", name.local_name ))); } XmlEvent::EndElement { name } => { assert!(name.local_name == "part"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(IfMessagePart::Part { file: file.to_string(), messages: Vec::new(), }) } fn parse_placeholder_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let name = get_attribute(attributes, "name")?; let mut content = String::new(); let mut example: Option = None; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "ex" => { if example.is_some() { return Err(Error::msg("Multiple examples in placeholder")); } example = Some(parse_placeholder_example_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in file", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "ph"); break; } XmlEvent::Characters(data) => { if example.is_some() { return Err(Error::msg("Text after example in placeholder")); } content.push_str(data.as_str()); } XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(TextPlaceholder::Placeholder { name: name.to_string(), content, example, }) } fn parse_placeholder_example_element( _attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let mut content = String::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes: _, namespace: _, } => { return Err(Error::msg(format!("Unexpected {0} in ex", name.local_name))); } XmlEvent::EndElement { name } => { assert!(name.local_name == "ex"); break; } XmlEvent::Characters(data) => { content.push_str(data.as_str()); } XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(content) } fn parse_grit_part_element( _attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let mut messages = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "message" => { messages.push(IfMessage::Message(parse_message_element( &attributes, reader, )?)); } "if" => { messages.push(parse_if_message_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in grit-part", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "grit-part"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(GritPart { messages }) } pub async fn parse_grit(path: impl AsRef) -> anyhow::Result { let path = path.as_ref().to_path_buf(); parse_grit_with_opener(move || Ok(BufReader::new(fs::File::open(path)?))).await } pub async fn parse_grit_with_opener(opener: F) -> anyhow::Result where F: FnOnce() -> anyhow::Result> + Send + 'static, R: Read, { spawn_blocking(move || { let reader = opener()?; let mut ereader = ParserConfig::new() .ignore_comments(true) .whitespace_to_characters(true) .cdata_to_characters(true) .create_reader(reader); let mut ret: Option = None; loop { let event = ereader.next()?; match event { XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::StartElement { name, attributes, namespace: _, } => { if name.local_name == "grit" { ret = Some(parse_grit_element(&attributes, &mut ereader)?); } else { return Err(Error::msg("Document root != grit")); } } XmlEvent::EndDocument => break, XmlEvent::EndElement { name: _ } => panic!("Unexpected EoE"), XmlEvent::Characters(_) => (), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(ret.unwrap()) }) .await .unwrap() } pub async fn parse_grit_part(path: impl AsRef) -> anyhow::Result { let path = path.as_ref().to_path_buf(); parse_grit_part_with_opener(|| Ok(BufReader::new(fs::File::open(path)?))).await } pub async fn parse_grit_part_with_opener(opener: F) -> anyhow::Result where F: FnOnce() -> anyhow::Result> + Send + 'static, R: Read, { spawn_blocking(move || { let reader = opener()?; let mut ereader = ParserConfig::new() .ignore_comments(true) .whitespace_to_characters(true) .cdata_to_characters(true) .create_reader(reader); let mut ret: Option = None; loop { let event = ereader.next()?; match event { XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::StartElement { name, attributes, namespace: _, } => { if name.local_name == "grit-part" { ret = Some(parse_grit_part_element(&attributes, &mut ereader)?); } else { return Err(Error::msg("Document root != grit-part")); } } XmlEvent::EndDocument => break, XmlEvent::EndElement { name: _ } => panic!("Unexpected EoE"), XmlEvent::Characters(_) => (), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(ret.unwrap()) }) .await .unwrap() } fn if_message_to_if_message_part(messages: Vec) -> Vec { messages .into_iter() .map(|x| match x { IfMessage::Message(message) => IfMessagePart::Message(message), IfMessage::If { expr, message } => IfMessagePart::If { expr, message: if_message_to_if_message_part(message), }, }) .collect() } async fn maybe_expand_message(message: &mut IfMessagePart, opener: &F) -> anyhow::Result<()> where F: Fn(&str) -> anyhow::Result> + Clone + Send + 'static, R: Read, { match message { IfMessagePart::Message(_) => Ok(()), IfMessagePart::Part { file, ref mut messages, } => { let file = file.to_string(); let opener = opener.clone(); let grit_part = parse_grit_part_with_opener(move || opener(file.as_str())).await?; *messages = if_message_to_if_message_part(grit_part.messages); Ok(()) } IfMessagePart::If { expr: _, ref mut message, } => { Box::pin(expand_messages(message, opener)).await?; Ok(()) } } } async fn expand_messages(messages: &mut Vec, opener: &F) -> anyhow::Result<()> where F: Fn(&str) -> anyhow::Result> + Clone + Send + 'static, R: Read, { for message in messages { maybe_expand_message(message, opener).await?; } Ok(()) } pub async fn parse_grit_with_parts(path: impl AsRef) -> anyhow::Result { let path = path.as_ref(); if let Some(basepath) = path.parent() { let basepath = basepath.to_path_buf(); parse_grit_with_parts_and_resolver(path, move |x| basepath.join(x)).await } else { parse_grit_with_parts_and_resolver(path, |x| PathBuf::from(x)).await } } pub async fn parse_grit_with_parts_and_resolver( path: impl AsRef, path_resolver: F, ) -> anyhow::Result where F: Fn(&str) -> PathBuf + Send + Clone + 'static, { let path = path.as_ref().to_path_buf(); let grit_opener = || Ok(BufReader::new(fs::File::open(path)?)); let part_opener = move |x: &str| { let part_path = path_resolver(x); Ok(BufReader::new(fs::File::open(part_path)?)) }; parse_grit_with_parts_and_opener(grit_opener, part_opener).await } pub async fn parse_grit_with_parts_and_opener( grit_opener: F, part_opener: G, ) -> anyhow::Result where F: FnOnce() -> anyhow::Result> + Send + 'static, G: Fn(&str) -> anyhow::Result> + Clone + Send + 'static, R: Read, { let mut grit = parse_grit_with_opener(grit_opener).await?; expand_messages(&mut grit.release.messages.messages, &part_opener).await?; Ok(grit) } fn get_presentable_content(message: &Message) -> String { let mut ret = String::new(); for c in &message.content { match c { TextPlaceholder::Text(value) => ret += value.as_str(), TextPlaceholder::Placeholder { name, content: _, example: _, } => ret += name.as_str(), } } ret } fn fingerprint(value: &str) -> i64 { u64::from_be_bytes(md5::compute(value).0[0..8].try_into().unwrap()).cast_signed() } pub fn get_message_id(message: &Message) -> i64 { let mut message_id = fingerprint(get_presentable_content(message).as_str()); if let Some(meaning) = &message.meaning { let meaning_id = fingerprint(meaning.as_str()); if message_id < 0 { message_id = (message_id << 1) + meaning_id + 1 } else { message_id = (message_id << 1) + meaning_id } } // Avoid returning negative ids message_id & 0x7fffffffffffffff } fn parse_translation_unit_target_element( _attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result> { let mut content = Vec::::new(); let mut first = true; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "ph" => { first = false; content.push(parse_translation_placeholder_element(&attributes, reader)?); } _ => { return Err(Error::msg(format!( "Unexpected {0} in file", name.local_name ))); } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "target"); break; } XmlEvent::Characters(data) => content.push(TextPlaceholder::Text(if first { first = false; data.trim_start().to_string() } else { data })), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } if !first { match content.last_mut().unwrap() { TextPlaceholder::Text(data) => { data.truncate(data.trim_end().len()); if data.is_empty() { content.pop(); } } TextPlaceholder::Placeholder { name: _, content: _, example: _, } => {} } } Ok(content) } fn parse_translation_placeholder_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let id = get_attribute(attributes, "id")?; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes: _, namespace: _, } => { return Err(Error::msg(format!("Unexpected {0} in ph", name.local_name))); } XmlEvent::EndElement { name } => { assert!(name.local_name == "ph"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(TextPlaceholder::Placeholder { name: id.to_string(), content: String::new(), example: None, }) } fn parse_translation_unit_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let id = get_attribute(attributes, "id")?.parse::()?; let mut target: Option> = None; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "target" => { if target.is_some() { return Err(Error::msg("Two target in trans-unit")); } target = Some(parse_translation_unit_target_element(&attributes, reader)?); } _ => { reader.skip()?; } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "trans-unit"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(TranslationUnit { id, target: target.expect("No target in trans-unit"), }) } fn parse_translation_body_element( _attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result> { let mut units = Vec::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "trans-unit" => { units.push(parse_translation_unit_element(&attributes, reader)?); } _ => { reader.skip()?; } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "body"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(units) } fn parse_translation_file_element( attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let target_language = get_attribute(attributes, "target-language")?; let mut units: Option> = None; loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => match name.local_name.as_str() { "body" => { if units.is_some() { return Err(Error::msg("More than one body in file")); } units = Some(parse_translation_body_element(&attributes, reader)?); } _ => { reader.skip()?; } }, XmlEvent::EndElement { name } => { assert!(name.local_name == "file"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(TranslationFile { target_language: target_language.to_string(), units: units.expect("body element in file"), }) } fn parse_xliff_element( _attributes: &[OwnedAttribute], reader: &mut EventReader, ) -> anyhow::Result { let mut file = VecDeque::::new(); loop { let event = reader.next()?; match event { XmlEvent::StartElement { name, attributes, namespace: _, } => { if name.local_name == "file" { file.push_back(parse_translation_file_element(&attributes, reader)?); } } XmlEvent::EndElement { name } => { assert!(name.local_name == "xliff"); break; } XmlEvent::Characters(_) => (), XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::EndDocument => panic!("Unexpected EOD"), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } if file.is_empty() { Err(Error::msg("No file in xliff")) } else if file.len() == 1 { Ok(file.pop_front().unwrap()) } else { let mut ret = file.pop_front().unwrap(); while !file.is_empty() { let other = file.pop_front().unwrap(); if other.target_language == ret.target_language { let end = ret.units.len(); ret.units.splice(end..end, other.units); } else { return Err(Error::msg( "Multiple translations in the same file, not supported yet", )); } } Ok(ret) } } pub async fn parse_xlf(path: impl AsRef) -> anyhow::Result { let path = path.as_ref().to_path_buf(); parse_xlf_with_opener(|| Ok(BufReader::new(fs::File::open(path)?))).await } pub async fn parse_xlf_with_opener(opener: F) -> anyhow::Result where F: FnOnce() -> anyhow::Result> + Send + 'static, R: Read, { spawn_blocking(move || { let reader = opener()?; let mut ereader = ParserConfig::new() .ignore_comments(true) .whitespace_to_characters(true) .cdata_to_characters(true) .create_reader(reader); let mut ret: Option = None; loop { let event = ereader.next()?; match event { XmlEvent::StartDocument { version: _, encoding: _, standalone: _, } => (), XmlEvent::StartElement { name, attributes, namespace: _, } => { if name.local_name == "xliff" { ret = Some(parse_xliff_element(&attributes, &mut ereader)?); } else { return Err(Error::msg("Document root != xliff")); } } XmlEvent::EndDocument => break, XmlEvent::EndElement { name: _ } => panic!("Unexpected EoE"), XmlEvent::Characters(_) => (), XmlEvent::ProcessingInstruction { name: _, data: _ } => (), XmlEvent::CData(_) => (), XmlEvent::Comment(_) => (), XmlEvent::Whitespace(_) => (), } } Ok(ret.unwrap()) }) .await .unwrap() }