diff --git a/nemo_text_processing/text_normalization/it/verbalizers/time.py b/nemo_text_processing/text_normalization/it/verbalizers/time.py index 6f098f152..b6e71ad28 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/time.py @@ -22,9 +22,11 @@ class TimeFst(GraphFst): """ Finite state transducer for verbalizing time, e.g. - tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e mezza trenta secondi - tokens { time { hours: "12" minutes: "15" } } -> dodici e un quarto + tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e trenta minuti e trenta secondi + tokens { time { hours: "12" minutes: "15" } } -> dodici e quindici minuti tokens { time { hours: "3" minutes: "38" } } -> tre e trentotto minuti + Deterministic verbalization prefers numeric minutes; nondeterministic verbalization + also provides equally weighted quarter-hour and half-hour alternatives. Args: cardinal_tagger: cardinal_tagger tagger GraphFst deterministic: if True will provide a single transduction option, @@ -41,11 +43,16 @@ def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True): minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + minute_verbalized = minute @ number_verbalization + pynutil.insert(" minuti") + colloquial_weight = 0.001 if deterministic else 0.0 + quarter_hour = pynutil.add_weight(minute @ pynini.cross("15", "un quarto"), colloquial_weight) + half_hour = pynutil.add_weight(minute @ pynini.cross("30", " mezza"), colloquial_weight) + graph_hms_15 = ( hour_verbalized + pynini.accep(" ") + pynutil.insert("e ") - + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) + + (minute_verbalized | quarter_hour) + pynini.accep(" ") + pynutil.insert("e ") + second @ number_verbalization @@ -56,26 +63,16 @@ def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True): hour_verbalized + pynini.accep(" ") + pynutil.insert("e ") - + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) + + (minute_verbalized | half_hour) + pynini.accep(" ") + pynutil.insert("e ") + second @ number_verbalization + pynutil.insert(" secondi") ) - graph_hm_15 = ( - hour_verbalized - + pynini.accep(" ") - + pynutil.insert("e ") - + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) - ) + graph_hm_15 = hour_verbalized + pynini.accep(" ") + pynutil.insert("e ") + (minute_verbalized | quarter_hour) - graph_hm_30 = ( - hour_verbalized - + pynini.accep(" ") - + pynutil.insert("e ") - + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) - ) + graph_hm_30 = hour_verbalized + pynini.accep(" ") + pynutil.insert("e ") + (minute_verbalized | half_hour) graph_h = hour_verbalized diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_electronic_post_process.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_electronic_post_process.txt new file mode 100644 index 000000000..fa29b6bf2 --- /dev/null +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_electronic_post_process.txt @@ -0,0 +1,8 @@ +Trovateci su mail.university.edu.~Trovateci su m a i l punto u n i v e r s i t y punto edu. +Trovateci su mail.nasa.gov.~Trovateci su m a i l punto n a s a punto gov. +Trovateci su mail.nasa.mx.~Trovateci su m a i l punto n a s a punto m x. +visitate il sito https://www.nvidia.com.~visitate il sito h t t p s due punti slash slash w w w punto nvidia punto com. +https://www.nvidia.com~h t t p s due punti slash slash w w w punto nvidia punto com +https://www.nvidia.com/abc/ciao.html.~h t t p s due punti slash slash w w w punto nvidia punto com slash a b c slash c i a o punto html. +abc.def@nvidia.com.~a b c punto d e f chiocciola nvidia punto com. +@jensen.me.~chiocciola j e n s e n punto m e. diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt index cc8e7667c..abfe6a28f 100644 --- a/tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt @@ -3,4 +3,6 @@ 17:15:26~diciassette e quindici minuti e ventisei secondi~diciassette e un quarto e ventisei secondi 23:45~ventitre e quarantacinque minuti 03:38~tre e trentotto minuti -l'evento inizia alle 16:00~l'evento inizia alle sedici \ No newline at end of file +l'evento inizia alle 16:00~l'evento inizia alle sedici +12:15~dodici e quindici minuti~dodici e un quarto +15:30:30~quindici e trenta minuti e trenta secondi~quindici e mezza e trenta secondi diff --git a/tests/nemo_text_processing/it/test_electronic.py b/tests/nemo_text_processing/it/test_electronic.py index bf9390883..09385abc0 100644 --- a/tests/nemo_text_processing/it/test_electronic.py +++ b/tests/nemo_text_processing/it/test_electronic.py @@ -27,5 +27,12 @@ class TestChar: @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm_char(self, test_input, expected): + preds = self.normalizer.normalize(test_input, punct_post_process=False) + assert expected == preds + + @parameterized.expand(parse_test_case_file('it/data_text_normalization/test_cases_electronic_post_process.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_char_punct_post_process(self, test_input, expected): preds = self.normalizer.normalize(test_input, punct_post_process=True) assert expected == preds diff --git a/tests/nemo_text_processing/it/test_time.py b/tests/nemo_text_processing/it/test_time.py index 8bb0eb1ab..3ceaa14f7 100644 --- a/tests/nemo_text_processing/it/test_time.py +++ b/tests/nemo_text_processing/it/test_time.py @@ -12,14 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pynini import pytest from parameterized import parameterized +from nemo_text_processing.text_normalization.it.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.it.verbalizers.time import TimeFst from nemo_text_processing.text_normalization.normalize import Normalizer from ..utils import CACHE_DIR, parse_test_case_file +@pytest.fixture(scope='module', params=[True, False]) +def time_verbalizer(request): + deterministic = request.param + cardinal = CardinalFst(deterministic=deterministic) + return deterministic, TimeFst(cardinal_tagger=cardinal, deterministic=deterministic).fst + + class TestChar: normalizer = Normalizer(lang='it', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') @@ -28,4 +38,36 @@ class TestChar: @pytest.mark.unit def test_norm_char(self, test_input, expected): preds = self.normalizer.normalize(test_input, punct_post_process=True) - assert preds in expected + assert preds == (expected[0] if isinstance(expected, list) else expected) + + @pytest.mark.parametrize( + 'tagged, expected, alternative', + [ + ('time { hours: "12" minutes: "15" }', 'dodici e quindici minuti', 'dodici e un quarto'), + ('time { hours: "12" minutes: "30" }', 'dodici e trenta minuti', 'dodici e mezza'), + ( + 'time { hours: "17" minutes: "15" seconds: "26" preserve_order: true }', + 'diciassette e quindici minuti e ventisei secondi', + 'diciassette e un quarto e ventisei secondi', + ), + ( + 'time { hours: "15" minutes: "30" seconds: "30" preserve_order: true }', + 'quindici e trenta minuti e trenta secondi', + 'quindici e mezza e trenta secondi', + ), + ], + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_time_verbalizer_priorities(self, time_verbalizer, tagged, expected, alternative): + deterministic, grammar = time_verbalizer + lattice = pynini.compose(tagged, grammar) + output_weights = {} + for _, output, weight in lattice.paths().items(): + output = ' '.join(output.split()) + output_weights[output] = min(output_weights.get(output, float('inf')), float(weight)) + assert set(output_weights) == {expected, alternative} + if deterministic: + assert output_weights[expected] < output_weights[alternative] + else: + assert output_weights[expected] == output_weights[alternative]