Update tokenize_and_classify.py

Jenkinsfile

@@ -18,7 +18,7 @@ pipeline {
     HI_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-4'
     FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0'
     HU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-16-24-0'
-    PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/05-01-26-1'
+    PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-24-26-1'
     RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
     VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-29-25-0'
     SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'

nemo_text_processing/text_normalization/data_loader_utils.py

@@ -47,6 +47,8 @@
     "ADDRESS",
     "ROMAN",
     "RANGE",
+    "SERIAL",
+    "WHITELIST",
 ]
 
 

nemo_text_processing/text_normalization/normalize.py

@@ -191,7 +191,11 @@ def __init__(
                 self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache)
         elif lang == 'pt':
             from nemo_text_processing.text_normalization.pt.taggers.tokenize_and_classify import ClassifyFst
+            from nemo_text_processing.text_normalization.pt.verbalizers.post_processing import PostProcessingFst
             from nemo_text_processing.text_normalization.pt.verbalizers.verbalize_final import VerbalizeFinalFst
+
+            if post_process:
+                self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache)
         elif lang == 'ko':
             from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst
             from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst
@@ -396,7 +400,7 @@ def normalize(
         output = SPACE_DUP.sub(' ', output[1:])
 
         if (
-            self.lang in ["en", "hi", "ja", "vi"]
+            self.lang in ["en", "hi", "ja", "vi", "pt"]
             and hasattr(self, 'post_processor')
             and self.post_processor is not None
         ):

nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py

@@ -84,7 +84,12 @@ def __init__(
             ordinal = OrdinalFst(cardinal, deterministic=deterministic)
             fraction = FractionFst(cardinal, ordinal, deterministic=deterministic)
             decimal = DecimalFst(cardinal, deterministic=deterministic)
-            measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic)
+            measure = MeasureFst(
+                cardinal=cardinal,
+                decimal=decimal,
+                fraction=fraction,
+                deterministic=deterministic,
+            )
             money = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic)
             date = DateFst(cardinal, deterministic=deterministic)
             time = TimeFst(cardinal, deterministic=deterministic)
@@ -110,21 +115,32 @@ def __init__(
                 | pynutil.add_weight(word_graph, 100)
             )
 
-            # Wrap tokens properly
-            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
-            punct_graph = (
+            punct = (
                 pynutil.insert("tokens { ") + pynutil.add_weight(punctuation.fst, weight=2.1) + pynutil.insert(" }")
             )
+            punct = pynini.closure(
+                pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space)
+                | (pynutil.insert(" ") + punct),
+                1,
+            )
+
+            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
+            token_plus_punct = (
+                pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct)
+            )
 
-            # Simple graph structure
-            graph = token + pynini.closure(
-                pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + token
+            graph = token_plus_punct + pynini.closure(
+                (
+                    pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space)
+                    | (pynutil.insert(" ") + punct + pynutil.insert(" "))
+                )
+                + token_plus_punct
             )
 
-            # Allow punctuation
-            graph |= punct_graph
+            graph = delete_space + graph + delete_space
+            graph |= punct
 
-            self.fst = delete_space + graph + delete_space
+            self.fst = graph.optimize()
 
             if far_file:
                 generator_main(far_file, {"tokenize_and_classify": self.fst})

nemo_text_processing/text_normalization/pt/verbalizers/post_processing.py

@@ -0,0 +1,78 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import pynini
+from pynini.lib import pynutil
+
+from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_SIGMA, NEMO_SPACE, generator_main
+from nemo_text_processing.utils.logging import logger
+
+
+class PostProcessingFst:
+    """
+    Finite state transducer that post-processes an entire Portuguese sentence after verbalization is complete, e.g.
+    removes extra spaces around punctuation marks " ( cento e vinte e três ) " -> "(cento e vinte e três)"
+
+    Args:
+        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
+        overwrite_cache: set to True to overwrite .far files
+    """
+
+    def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            far_file = os.path.join(cache_dir, "pt_tn_post_processing.far")
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["post_process_graph"]
+            logger.info(f'Post processing graph was restored from {far_file}.')
+        else:
+            self.fst = self.get_punct_postprocess_graph()
+
+            if far_file:
+                generator_main(far_file, {"post_process_graph": self.fst})
+
+    def get_punct_postprocess_graph(self):
+        """
+        Returns graph to post process punctuation marks for Portuguese, removing the spaces
+        that the tokenizer/verbalizer unconditionally inserts between tokens.
+        """
+        no_space_before_punct = [",", ".", "!", "?", ":", ";", ")", r"\]", "}"]
+        no_space_after_punct = ["(", r"\[", "{"]
+
+        no_space_before_punct_fst = pynini.union(*no_space_before_punct)
+        no_space_after_punct_fst = pynini.union(*no_space_after_punct)
+
+        delete_space = pynutil.delete(NEMO_SPACE)
+
+        # "word ," -> "word,"
+        remove_space_before = pynini.cdrewrite(
+            delete_space + no_space_before_punct_fst,
+            "",
+            "",
+            NEMO_SIGMA,
+        ).optimize()
+
+        # "( word" -> "(word"
+        remove_space_after = pynini.cdrewrite(
+            no_space_after_punct_fst + delete_space,
+            "",
+            "",
+            NEMO_SIGMA,
+        ).optimize()
+
+        graph = pynini.compose(remove_space_before, remove_space_after)
+        return graph.optimize()

nemo_text_processing/text_normalization/run_evaluate.py

@@ -35,7 +35,7 @@ def parse_args():
     parser.add_argument(
         "--lang",
         help="language",
-        choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'],
+        choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt', 'ja'],
         default="en",
         type=str,
     )

tests/nemo_text_processing/pt/data_text_normalization/test_cases_punctuation.txt

@@ -0,0 +1,20 @@
+A versão beta sai às 17:00; a compilação 42 está pronta.~A versão beta sai às dezessete horas; a compilação quarenta e dois está pronta.
+Ela terminou em 1.º, depois em 23.º e por fim em 100.º.~Ela terminou em primeiro, depois em vigésimo terceiro e por fim em centésimo.
+O preço é R$ 4,99 e o outro valor é € 1.234,78.~O preço é quatro reais e noventa e nove centavos e o outro valor é euro um ponto dois três quatro vírgula sete oito.
+A reunião começa às 14:00 e termina às 23:59.~A reunião começa às catorze horas e termina às vinte e três horas e cinquenta e nove.
+O resultado foi 95% e depois 3,5%.~O resultado foi noventa e cinco por cento e depois três vírgula cinco por cento.
+301(a)~trezentos e um (a)
+42(b)(1)~quarenta e dois (b)(um)
+(42)~(quarenta e dois)
+42;~quarenta e dois;
+42,,99~quarenta e dois,, noventa e nove
+42...~quarenta e dois...
+42!!~quarenta e dois!!
+R$4,99.~quatro reais e noventa e nove centavos.
+A consulta é em 20/05/2023 e 31/12/2025.~A consulta é em vinte de maio de dois mil e vinte e três e trinta e um de dezembro de dois mil e vinte e cinco.
+O evento começa em 01/01/2025 às 14:30.~O evento começa em primeiro de janeiro de dois mil e vinte e cinco às catorze horas e trinta.
+O valor de pi é 3,14159.~O valor de pi é três vírgula um quatro um cinco nove.
+Instale v1.5.3 depois da versão 2.0.~Instale v um ponto cinco ponto três depois da versão dois ponto zero.
+O número romano MCMXC representa 1990.~O número romano MCMXC representa mil novecentos e noventa.
+Ligue para +55 11 91234-5678.~Ligue para mais cinco cinco um um nove um dois três quatro cinco seis sete oito.
+Consulte § 301(a) e 42 U.S.C. § 1983.~Consulte parágrafo trezentos e um (a) e quarenta e dois USC parágrafo mil novecentos e oitenta e três.

tests/nemo_text_processing/pt/test_punctuation.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from parameterized import parameterized
+
+from nemo_text_processing.text_normalization.normalize import Normalizer
+
+from ..utils import CACHE_DIR, parse_test_case_file
+
+
+class TestPunctuation:
+    normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased')
+
+    @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_punctuation.txt'))
+    @pytest.mark.run_only_on('CPU')
+    @pytest.mark.unit
+    def test_norm(self, test_input, expected):
+        pred = self.normalizer.normalize(test_input, verbose=False)
+        assert pred == expected

tests/nemo_text_processing/pt/test_sparrowhawk_normalization.sh

@@ -16,7 +16,7 @@ runtest () {
     spoken=${testcase_tokenized[1]}
 
     # replace non breaking space with breaking space
-    denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g')
+    denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration_pp.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g')
 
     # trim white space
     spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
@@ -82,6 +82,11 @@ testTNElectronic() {
   runtest $input
 }
 
+testTNPunctuation() {
+  input=$PROJECT_DIR/pt/data_text_normalization/test_cases_punctuation.txt
+  runtest $input
+}
+
 # testTNWord() {
 #   input=$PROJECT_DIR/pt/data_text_normalization/test_cases_word.txt
 #   runtest $input

tools/text_processing_deployment/pynini_export.py

@@ -198,6 +198,9 @@ def parse_args():
         from nemo_text_processing.text_normalization.pt.taggers.tokenize_and_classify import (
             ClassifyFst as TNClassifyFst,
         )
+        from nemo_text_processing.text_normalization.pt.verbalizers.post_processing import (
+            PostProcessingFst as TNPostProcessingFst,
+        )
         from nemo_text_processing.text_normalization.pt.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst
     elif args.language == 'fr':
         from nemo_text_processing.inverse_text_normalization.fr.taggers.tokenize_and_classify import (