diff --git a/README.md b/README.md index 8bbb5c617..7a330462d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,8 @@ pip install nemo_text_processing **_NOTE:_** This should work on any Linux OS with x86_64. Pip installation on MacOS and Windows are not supported due to the dependency [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini). On a platform other than Linux x86_64, installing from Pip tries to compile Pynini from scratch, and requires OpenFst headers and libraries to be in the expected place. So if it's working for you, it's because you happen to have installed OpenFst in the right way in the right place. So if you want to Pip install Pynini on MacOS, you have to have pre-compiled and pre-installed OpenFst. The Pynini README for that version should tell you which version it needs and what `--enable-foo` flags to use. Instead, we recommend you to use conda-forge to install Pynini on MacOS or Windows: -`conda install -c conda-forge pynini=2.1.6.post1`. +`conda install -c conda-forge "pynini<=2.1.7"`. +For Python 3.8, use `conda install -c conda-forge pynini=2.1.6.post1` instead. ### Pip from source diff --git a/nemo_text_processing/text_normalization/en/taggers/electronic.py b/nemo_text_processing/text_normalization/en/taggers/electronic.py index 25c3c445a..6c046190a 100644 --- a/nemo_text_processing/text_normalization/en/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/en/taggers/electronic.py @@ -18,6 +18,7 @@ from nemo_text_processing.text_normalization.en.graph_utils import ( # common string literals MIN_NEG_WEIGHT, + MIN_POS_WEIGHT, NEMO_ALPHA, NEMO_DIGIT, NEMO_NOT_SPACE, @@ -76,12 +77,19 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ) dict_words_graph = dict_words_without_delimiter | dict_words + # Prefer complete dictionary words over partial matches such as "proGRAM". + spelled_letters = pynutil.add_weight(TO_UPPER, MIN_POS_WEIGHT) if deterministic else TO_UPPER + all_accepted_symbols_start = ( - dict_words_graph | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + dict_words_graph | pynini.closure(spelled_letters) | pynini.closure(NEMO_UPPER) | accepted_symbols ).optimize() all_accepted_symbols_end = ( - dict_words_graph | numbers | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols + dict_words_graph + | numbers + | pynini.closure(spelled_letters) + | pynini.closure(NEMO_UPPER) + | accepted_symbols ).optimize() graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() diff --git a/nemo_text_processing/text_normalization/en/taggers/serial.py b/nemo_text_processing/text_normalization/en/taggers/serial.py index 4a5d6bb9d..d42b22c99 100644 --- a/nemo_text_processing/text_normalization/en/taggers/serial.py +++ b/nemo_text_processing/text_normalization/en/taggers/serial.py @@ -180,7 +180,11 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = ).optimize() serial_graph |= pynutil.add_weight(slash_serial, MIN_NEG_WEIGHT) - serial_graph |= pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize() + serial_graph_alnum = pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize() + if deterministic: + # Prefer the delimited reading when both paths match, e.g. "133-ABC". + serial_graph_alnum = pynutil.add_weight(serial_graph_alnum, MIN_POS_WEIGHT) + serial_graph |= serial_graph_alnum serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize() # this is not to verbolize "/" as "slash" in cases like "import/export" diff --git a/nemo_text_processing/text_normalization/vi/taggers/fraction.py b/nemo_text_processing/text_normalization/vi/taggers/fraction.py index c54f5be5f..5e72e77e9 100644 --- a/nemo_text_processing/text_normalization/vi/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/vi/taggers/fraction.py @@ -48,6 +48,9 @@ def __init__(self, cardinal: CardinalFst, deterministic: bool = True): denominator_exception_graph = ( pynini.union(*denominator_exception_patterns) if denominator_exception_patterns else None ) + if denominator_exception_graph and deterministic: + # Prefer denominator exceptions over cardinal readings in every decoder. + denominator_exception_graph = pynutil.add_weight(denominator_exception_graph, -0.0001) denominator_graph = ( pynini.union(denominator_exception_graph, cardinal_graph) if denominator_exception_graph diff --git a/nemo_text_processing/text_normalization/vi/taggers/ordinal.py b/nemo_text_processing/text_normalization/vi/taggers/ordinal.py index af8ca3053..1f25f1793 100644 --- a/nemo_text_processing/text_normalization/vi/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/vi/taggers/ordinal.py @@ -49,6 +49,9 @@ def __init__(self, cardinal, deterministic: bool = True): combined_graph = cardinal.graph if exception_graph: + if deterministic: + # Prefer ordinal exceptions over cardinal readings in every decoder. + exception_graph = pynutil.add_weight(exception_graph, -0.0001) combined_graph = pynini.union(exception_graph, cardinal.graph) self.graph = ( diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6622e2d49..f0854ed15 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,7 +3,8 @@ editdistance inflect joblib pandas -pynini==2.1.6.post1 +pynini==2.1.6.post1; python_version < "3.9" +pynini<=2.1.7; python_version >= "3.9" regex sacremoses>=0.0.43 setuptools>=65.5.1 diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt index aacfde319..6d22134a6 100644 --- a/requirements/requirements_test.txt +++ b/requirements/requirements_test.txt @@ -2,7 +2,8 @@ black==25.1.0 click>=8.0.2 isort[requirements]>5.1.0,<=6.0.1 parameterized -pynini==2.1.6.post1 +pynini==2.1.6.post1; python_version < "3.9" +pynini<=2.1.7; python_version >= "3.9" pytest pytest-runner ruamel.yaml diff --git a/tests/nemo_text_processing/vi/test_fraction.py b/tests/nemo_text_processing/vi/test_fraction.py index efa35fcce..37e981641 100644 --- a/tests/nemo_text_processing/vi/test_fraction.py +++ b/tests/nemo_text_processing/vi/test_fraction.py @@ -14,6 +14,7 @@ import pytest from parameterized import parameterized +from pynini.lib import rewrite from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer @@ -43,6 +44,23 @@ def test_denorm(self, test_input, expected): else None ) + @parameterized.expand( + [ + ("1/4", "một phần tư"), + ("3/4", "ba phần tư"), + ("-3/4", "âm ba phần tư"), + ("5 1/4", "năm và một phần tư"), + ("3/4m", "ba phần tư mét"), + ("1/3", "một phần ba"), + ] + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_unique_exception(self, test_input, expected): + # A plain shortest path can hide ties that another decoder resolves differently. + tagged = rewrite.one_top_rewrite(test_input, self.normalizer.tagger.fst) + assert rewrite.one_top_rewrite(tagged, self.normalizer.verbalizer.fst) == expected + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_fraction.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit diff --git a/tests/nemo_text_processing/vi/test_ordinal.py b/tests/nemo_text_processing/vi/test_ordinal.py index 9b15bd0c4..2a244f92d 100644 --- a/tests/nemo_text_processing/vi/test_ordinal.py +++ b/tests/nemo_text_processing/vi/test_ordinal.py @@ -14,6 +14,7 @@ import pytest from parameterized import parameterized +from pynini.lib import rewrite from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer @@ -43,6 +44,14 @@ def test_denorm(self, test_input, expected): else None ) + @parameterized.expand([("thứ 1", "thứ nhất"), ("thứ 4", "thứ tư"), ("thứ 3", "thứ ba")]) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_unique_exception(self, test_input, expected): + # A plain shortest path can hide ties that another decoder resolves differently. + tagged = rewrite.one_top_rewrite(test_input, self.normalizer.tagger.fst) + assert rewrite.one_top_rewrite(tagged, self.normalizer.verbalizer.fst) == expected + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_ordinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit