Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ pip install nemo_text_processing

**_NOTE:_** This should work on any Linux OS with x86_64. Pip installation on MacOS and Windows are not supported due to the dependency [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini). On a platform other than Linux x86_64, installing from Pip tries to compile Pynini from scratch, and requires OpenFst headers and libraries to be in the expected place. So if it's working for you, it's because you happen to have installed OpenFst in the right way in the right place. So if you want to Pip install Pynini on MacOS, you have to have pre-compiled and pre-installed OpenFst. The Pynini README for that version should tell you which version it needs and what `--enable-foo` flags to use.
Instead, we recommend you to use conda-forge to install Pynini on MacOS or Windows:
`conda install -c conda-forge pynini=2.1.6.post1`.
`conda install -c conda-forge "pynini<=2.1.7"`.
For Python 3.8, use `conda install -c conda-forge pynini=2.1.6.post1` instead.


### Pip from source
Expand Down
12 changes: 10 additions & 2 deletions nemo_text_processing/text_normalization/en/taggers/electronic.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from nemo_text_processing.text_normalization.en.graph_utils import ( # common string literals
MIN_NEG_WEIGHT,
MIN_POS_WEIGHT,
NEMO_ALPHA,
NEMO_DIGIT,
NEMO_NOT_SPACE,
Expand Down Expand Up @@ -76,12 +77,19 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
)
dict_words_graph = dict_words_without_delimiter | dict_words

# Prefer complete dictionary words over partial matches such as "proGRAM".
spelled_letters = pynutil.add_weight(TO_UPPER, MIN_POS_WEIGHT) if deterministic else TO_UPPER

all_accepted_symbols_start = (
dict_words_graph | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols
dict_words_graph | pynini.closure(spelled_letters) | pynini.closure(NEMO_UPPER) | accepted_symbols
).optimize()

all_accepted_symbols_end = (
dict_words_graph | numbers | pynini.closure(TO_UPPER) | pynini.closure(NEMO_UPPER) | accepted_symbols
dict_words_graph
| numbers
| pynini.closure(spelled_letters)
| pynini.closure(NEMO_UPPER)
| accepted_symbols
).optimize()

graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize()
Expand Down
6 changes: 5 additions & 1 deletion nemo_text_processing/text_normalization/en/taggers/serial.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,11 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool =
).optimize()
serial_graph |= pynutil.add_weight(slash_serial, MIN_NEG_WEIGHT)

serial_graph |= pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize()
serial_graph_alnum = pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize()
if deterministic:
# Prefer the delimited reading when both paths match, e.g. "133-ABC".
serial_graph_alnum = pynutil.add_weight(serial_graph_alnum, MIN_POS_WEIGHT)
serial_graph |= serial_graph_alnum
serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize()

# this is not to verbolize "/" as "slash" in cases like "import/export"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ def __init__(self, cardinal: CardinalFst, deterministic: bool = True):
denominator_exception_graph = (
pynini.union(*denominator_exception_patterns) if denominator_exception_patterns else None
)
if denominator_exception_graph and deterministic:
# Prefer denominator exceptions over cardinal readings in every decoder.
denominator_exception_graph = pynutil.add_weight(denominator_exception_graph, -0.0001)
denominator_graph = (
pynini.union(denominator_exception_graph, cardinal_graph)
if denominator_exception_graph
Expand Down
3 changes: 3 additions & 0 deletions nemo_text_processing/text_normalization/vi/taggers/ordinal.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ def __init__(self, cardinal, deterministic: bool = True):

combined_graph = cardinal.graph
if exception_graph:
if deterministic:
# Prefer ordinal exceptions over cardinal readings in every decoder.
exception_graph = pynutil.add_weight(exception_graph, -0.0001)
combined_graph = pynini.union(exception_graph, cardinal.graph)

self.graph = (
Expand Down
3 changes: 2 additions & 1 deletion requirements/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ editdistance
inflect
joblib
pandas
pynini==2.1.6.post1
pynini==2.1.6.post1; python_version < "3.9"
pynini<=2.1.7; python_version >= "3.9"
regex
sacremoses>=0.0.43
setuptools>=65.5.1
Expand Down
3 changes: 2 additions & 1 deletion requirements/requirements_test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ black==25.1.0
click>=8.0.2
isort[requirements]>5.1.0,<=6.0.1
parameterized
pynini==2.1.6.post1
pynini==2.1.6.post1; python_version < "3.9"
pynini<=2.1.7; python_version >= "3.9"
pytest
pytest-runner
ruamel.yaml
Expand Down
18 changes: 18 additions & 0 deletions tests/nemo_text_processing/vi/test_fraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest
from parameterized import parameterized
from pynini.lib import rewrite

from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
Expand Down Expand Up @@ -43,6 +44,23 @@ def test_denorm(self, test_input, expected):
else None
)

@parameterized.expand(
[
("1/4", "một phần tư"),
("3/4", "ba phần tư"),
("-3/4", "âm ba phần tư"),
("5 1/4", "năm và một phần tư"),
("3/4m", "ba phần tư mét"),
("1/3", "một phần ba"),
]
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_unique_exception(self, test_input, expected):
# A plain shortest path can hide ties that another decoder resolves differently.
tagged = rewrite.one_top_rewrite(test_input, self.normalizer.tagger.fst)
assert rewrite.one_top_rewrite(tagged, self.normalizer.verbalizer.fst) == expected

@parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_fraction.txt'))
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
Expand Down
9 changes: 9 additions & 0 deletions tests/nemo_text_processing/vi/test_ordinal.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest
from parameterized import parameterized
from pynini.lib import rewrite

from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
Expand Down Expand Up @@ -43,6 +44,14 @@ def test_denorm(self, test_input, expected):
else None
)

@parameterized.expand([("thứ 1", "thứ nhất"), ("thứ 4", "thứ tư"), ("thứ 3", "thứ ba")])
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_unique_exception(self, test_input, expected):
# A plain shortest path can hide ties that another decoder resolves differently.
tagged = rewrite.one_top_rewrite(test_input, self.normalizer.tagger.fst)
assert rewrite.one_top_rewrite(tagged, self.normalizer.verbalizer.fst) == expected

@parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_ordinal.txt'))
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
Expand Down
Loading