From 2503a71ac1a9c0ad1b2f2efdc320658ad1f2c1bf Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Sun, 3 Sep 2023 14:12:19 +0000 Subject: [PATCH 1/3] =?UTF-8?q?recreate=20commits=20from=20#11=20(Northern?= =?UTF-8?q?=20S=C3=A1mi=20TN)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jim O'Regan --- Jenkinsfile | 22 + .../text_normalization/normalize.py | 5 +- .../text_normalization/se/__init__.py | 13 + .../text_normalization/se/data/__init__.py | 13 + .../se/data/dates/__init__.py | 13 + .../se/data/dates/era_words.tsv | 4 + .../se/data/dates/months_abbr.tsv | 13 + .../se/data/dates/numbers.tsv | 12 + .../se/data/electronic/__init__.py | 13 + .../se/data/electronic/domain.tsv | 13 + .../se/data/electronic/server_name.tsv | 18 + .../se/data/electronic/symbols.tsv | 21 + .../se/data/inflection/__init__.py | 13 + .../se/data/inflection/jahki.tsv | 11 + .../se/data/inflection/simple.tsv | 9 + .../se/data/inflection/vahkku.tsv | 10 + .../se/data/math_operations.tsv | 8 + .../se/data/measure/__init__.py | 13 + .../se/data/measure/greek_lower.tsv | 25 + .../se/data/measure/unit_simple.tsv | 13 + .../se/data/money/__init__.py | 13 + .../se/data/money/currency_major.tsv | 10 + .../se/data/money/currency_major_case.tsv | 9 + .../se/data/money/currency_major_gen.tsv | 10 + .../se/data/money/currency_minor.tsv | 10 + .../se/data/money/currency_minor_gen.tsv | 10 + .../se/data/numbers/__init__.py | 13 + .../se/data/numbers/case_cuodi.tsv | 11 + .../se/data/numbers/case_duhat.tsv | 11 + .../se/data/numbers/case_logi.tsv | 10 + .../se/data/numbers/case_miljon.tsv | 11 + .../se/data/numbers/case_nolla.tsv | 11 + .../se/data/numbers/digit.tsv | 9 + .../se/data/numbers/digit_bare_com_pl.tsv | 9 + .../se/data/numbers/digit_bare_com_sg.tsv | 9 + .../se/data/numbers/digit_bare_ess.tsv | 9 + .../se/data/numbers/digit_bare_gen_pl.tsv | 9 + .../se/data/numbers/digit_bare_gen_sg.tsv | 9 + .../se/data/numbers/digit_bare_ill_pl.tsv | 9 + .../se/data/numbers/digit_bare_ill_sg.tsv | 9 + .../se/data/numbers/digit_bare_loc_sg.tsv | 9 + .../se/data/numbers/digit_bare_nom_pl.tsv | 9 + .../data/numbers/digit_case_abbr_suffix.tsv | 9 + .../se/data/numbers/digit_com_pl.tsv | 9 + .../se/data/numbers/digit_com_sg.tsv | 9 + .../se/data/numbers/digit_ess.tsv | 9 + .../se/data/numbers/digit_gen_pl.tsv | 9 + .../se/data/numbers/digit_gen_sg.tsv | 9 + .../se/data/numbers/digit_ill_pl.tsv | 9 + .../se/data/numbers/digit_ill_sg.tsv | 9 + .../se/data/numbers/digit_loc_sg.tsv | 9 + .../se/data/numbers/digit_nom_pl.tsv | 9 + .../se/data/numbers/millions.tsv | 11 + .../se/data/numbers/zero.tsv | 1 + .../se/data/ordinal/__init__.py | 13 + .../se/data/ordinal/digit.tsv | 9 + .../se/data/ordinal/superscript_digits.tsv | 10 + .../text_normalization/se/data/symbols.tsv | 21 + .../se/data/telephone/__init__.py | 13 + .../se/data/telephone/ip_prompt.tsv | 6 + .../se/data/telephone/telephone_abbr.tsv | 4 + .../se/data/telephone/telephone_abbr_nd.tsv | 8 + .../se/data/telephone/telephone_prompt.tsv | 9 + .../se/data/time/__init__.py | 13 + .../se/data/time/prefix.tsv | 8 + .../se/data/time/suffix.tsv | 2 + .../se/data/time/time_zone.tsv | 7 + .../text_normalization/se/data/whitelist.tsv | 7 + .../text_normalization/se/graph_utils.py | 40 ++ .../text_normalization/se/taggers/__init__.py | 13 + .../se/taggers/abbreviation.py | 52 +++ .../text_normalization/se/taggers/cardinal.py | 434 ++++++++++++++++++ .../text_normalization/se/taggers/date.py | 126 +++++ .../text_normalization/se/taggers/decimal.py | 105 +++++ .../se/taggers/electronic.py | 70 +++ .../text_normalization/se/taggers/fraction.py | 96 ++++ .../text_normalization/se/taggers/measure.py | 212 +++++++++ .../text_normalization/se/taggers/money.py | 232 ++++++++++ .../text_normalization/se/taggers/ordinal.py | 192 ++++++++ .../se/taggers/telephone.py | 140 ++++++ .../text_normalization/se/taggers/time.py | 217 +++++++++ .../se/taggers/tokenize_and_classify.py | 188 ++++++++ .../se/taggers/whitelist.py | 64 +++ .../text_normalization/se/taggers/word.py | 32 ++ .../text_normalization/se/utils.py | 72 +++ .../se/verbalizers/__init__.py | 13 + .../se/verbalizers/cardinal.py | 42 ++ .../text_normalization/se/verbalizers/date.py | 82 ++++ .../se/verbalizers/decimals.py | 73 +++ .../se/verbalizers/electronic.py | 74 +++ .../se/verbalizers/fraction.py | 73 +++ .../se/verbalizers/measure.py | 95 ++++ .../se/verbalizers/money.py | 72 +++ .../se/verbalizers/ordinal.py | 46 ++ .../se/verbalizers/telephone.py | 73 +++ .../text_normalization/se/verbalizers/time.py | 89 ++++ .../se/verbalizers/verbalize.py | 73 +++ .../se/verbalizers/verbalize_final.py | 70 +++ tests/nemo_text_processing/se/__init__.py | 13 + .../se/data_text_normalization/__init__.py | 13 + .../test_cases_cardinal.txt | 62 +++ .../test_cases_date.txt | 1 + .../test_cases_electronic.txt | 12 + .../test_cases_ordinal.txt | 38 ++ .../test_cases_time.txt | 2 + .../test_cases_whitelist.txt | 1 + .../test_cases_word.txt | 48 ++ .../nemo_text_processing/se/test_cardinal.py | 30 ++ tests/nemo_text_processing/se/test_date.py | 30 ++ .../se/test_electronic.py | 30 ++ tests/nemo_text_processing/se/test_ordinal.py | 31 ++ tests/nemo_text_processing/se/test_time.py | 30 ++ .../nemo_text_processing/se/test_whitelist.py | 30 ++ tests/nemo_text_processing/se/test_word.py | 30 ++ 114 files changed, 4273 insertions(+), 1 deletion(-) create mode 100644 nemo_text_processing/text_normalization/se/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/dates/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/dates/era_words.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/dates/months_abbr.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/dates/numbers.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/electronic/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/electronic/domain.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/electronic/server_name.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/inflection/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/inflection/jahki.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/inflection/simple.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/inflection/vahkku.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/math_operations.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/measure/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/measure/greek_lower.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/money/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/money/currency_major.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/money/currency_major_case.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/money/currency_minor.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/money/currency_minor_gen.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/case_cuodi.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/case_duhat.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/case_logi.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/case_miljon.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/case_nolla.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ess.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_loc_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_bare_nom_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_case_abbr_suffix.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_com_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_com_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_ess.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_gen_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_gen_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_ill_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_ill_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_loc_sg.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/digit_nom_pl.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/millions.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/zero.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/ordinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/ordinal/digit.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/ordinal/superscript_digits.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/symbols.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/telephone/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/telephone/ip_prompt.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr_nd.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/telephone/telephone_prompt.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/data/time/prefix.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/time/suffix.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/time/time_zone.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/whitelist.tsv create mode 100644 nemo_text_processing/text_normalization/se/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/abbreviation.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/decimal.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/electronic.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/fraction.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/money.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/telephone.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/se/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/se/utils.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/decimals.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/electronic.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/fraction.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/money.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/telephone.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py create mode 100644 tests/nemo_text_processing/se/__init__.py create mode 100644 tests/nemo_text_processing/se/data_text_normalization/__init__.py create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_electronic.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/se/test_cardinal.py create mode 100644 tests/nemo_text_processing/se/test_date.py create mode 100644 tests/nemo_text_processing/se/test_electronic.py create mode 100644 tests/nemo_text_processing/se/test_ordinal.py create mode 100644 tests/nemo_text_processing/se/test_time.py create mode 100644 tests/nemo_text_processing/se/test_whitelist.py create mode 100644 tests/nemo_text_processing/se/test_word.py diff --git a/Jenkinsfile b/Jenkinsfile index d09e25313..7c5663b74 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ pipeline { PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' + SE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-12-23-0' DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' @@ -287,6 +288,27 @@ pipeline { } } } + stage('L0: Create SE TN/ITN Grammars') { + when { + anyOf { + branch 'main' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L0: SE TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=se --text="100" --cache_dir ${SV_TN_CACHE}' + } + } + // stage('L0: SE ITN grammars') { + // steps { + // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=se --text="hundra " --cache_dir ${SV_TN_CACHE}' + // } + // } + } + } stage('L0: Create SV TN/ITN Grammars') { when { anyOf { diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index a2018cf80..4db992c21 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -144,6 +144,9 @@ def __init__( elif lang == 'fr': from nemo_text_processing.text_normalization.fr.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.fr.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'se': + from nemo_text_processing.text_normalization.se.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.se.verbalizers.verbalize_final import VerbalizeFinalFst elif lang == 'sv': from nemo_text_processing.text_normalization.sv.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.sv.verbalizers.verbalize_final import VerbalizeFinalFst @@ -691,7 +694,7 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it"], + choices=["en", "de", "es", "fr", "hu", "se", "sv", "zh", "ar", "it"], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/se/__init__.py b/nemo_text_processing/text_normalization/se/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/__init__.py b/nemo_text_processing/text_normalization/se/data/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/dates/__init__.py b/nemo_text_processing/text_normalization/se/data/dates/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/dates/era_words.tsv b/nemo_text_processing/text_normalization/se/data/dates/era_words.tsv new file mode 100644 index 000000000..6767e14db --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/dates/era_words.tsv @@ -0,0 +1,4 @@ +o.Kr. ovdal Kristtusa +m.Kr. maŋŋel Kristtusa +oKr. ovdal Kristtusa +mKr. maŋŋel Kristtusa diff --git a/nemo_text_processing/text_normalization/se/data/dates/months_abbr.tsv b/nemo_text_processing/text_normalization/se/data/dates/months_abbr.tsv new file mode 100644 index 000000000..dc0ff4bd6 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/dates/months_abbr.tsv @@ -0,0 +1,13 @@ +ođđajagem ođđajagemánnu +ođđajagim ođđajagimánnu +guovvam guovvamánnu +njukčam njukčamánnu +cuoŋom cuoŋománnu +miessem miessemánnu +geassem geassemánnu +suoidnem suoidnemánnu +borgem borgemánnu +čakcam čakčamánnu +golggotm golggotmánnu +skábmam skábmamánnu +juovlam juovlamánnu diff --git a/nemo_text_processing/text_normalization/se/data/dates/numbers.tsv b/nemo_text_processing/text_normalization/se/data/dates/numbers.tsv new file mode 100644 index 000000000..f16a17326 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/dates/numbers.tsv @@ -0,0 +1,12 @@ +1 ođđajagimánnu +2 guovvamánnu +3 njukčamánnu +4 cuoŋománnu +5 miessemánnu +6 geassemánnu +7 suoidnemánnu +8 borgemánnu +9 čakčamánnu +10 golggotmánnu +11 skábmamánnu +12 juovlamánnu diff --git a/nemo_text_processing/text_normalization/se/data/electronic/__init__.py b/nemo_text_processing/text_normalization/se/data/electronic/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/electronic/domain.tsv b/nemo_text_processing/text_normalization/se/data/electronic/domain.tsv new file mode 100644 index 000000000..e233e38e3 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/electronic/domain.tsv @@ -0,0 +1,13 @@ +.com čuokkis com +.uk čuokkis u k +.fr čuokkis f r +.net čuokkis net +.br čuokkis b r +.in čuokkis i n +.ru čuokkis r u +.de čuokkis d e +.it čuokkis i t +.se čuokkis s e +.no čuokkis n o +.fi čuokkis f i +.es čuokkis e s diff --git a/nemo_text_processing/text_normalization/se/data/electronic/server_name.tsv b/nemo_text_processing/text_normalization/se/data/electronic/server_name.tsv new file mode 100644 index 000000000..23b984360 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/electronic/server_name.tsv @@ -0,0 +1,18 @@ +gmail g mail +nvidia +outlook +hotmail +yahoo +live +yandex +orange +wanadoo +web +comcast +aol +sametinget +samediggi +sámediggi +ijahisidja ijahis idja +sajos +salliparikka salli parikka diff --git a/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv b/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv new file mode 100644 index 000000000..c86358fb5 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv @@ -0,0 +1,21 @@ +. čuokkis +: duppalčuokkis +- goallossáhcu +_ +! čuorvvasmearka +; beallečuokkis +# +$ +% +& +' apostrofa +* ++ +/ finjusáhcu += +? gažaldatmearka +^ +{ +} +~ +, rihkku diff --git a/nemo_text_processing/text_normalization/se/data/inflection/__init__.py b/nemo_text_processing/text_normalization/se/data/inflection/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/inflection/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/inflection/jahki.tsv b/nemo_text_processing/text_normalization/se/data/inflection/jahki.tsv new file mode 100644 index 000000000..21b072d53 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/inflection/jahki.tsv @@ -0,0 +1,11 @@ +ess jahkin +com_pl jagiiguin +com_sg jagiin +acc_sg jagi +gen_sg jage +gen_pl jagiid +ill_pl jagiide +ill_sg jahkái +loc_sg jagis +nom_pl jagit +nom_sg jagi diff --git a/nemo_text_processing/text_normalization/se/data/inflection/simple.tsv b/nemo_text_processing/text_normalization/se/data/inflection/simple.tsv new file mode 100644 index 000000000..7284ce9fe --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/inflection/simple.tsv @@ -0,0 +1,9 @@ +:t at +:a a +:id iid +:i ii +:ida iidda +:s is +:in iin +:iguin iiguin +:n in \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/inflection/vahkku.tsv b/nemo_text_processing/text_normalization/se/data/inflection/vahkku.tsv new file mode 100644 index 000000000..d3e40607a --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/inflection/vahkku.tsv @@ -0,0 +1,10 @@ +ess vahkkun +com_pl vahkuiguin +com_sg vahkuin +gen_sg vahku +gen_pl vahkuid +ill_pl vahkuide +ill_sg vahkkui +loc_sg vahkus +nom_pl vahkut +nom_sg vahkku diff --git a/nemo_text_processing/text_normalization/se/data/math_operations.tsv b/nemo_text_processing/text_normalization/se/data/math_operations.tsv new file mode 100644 index 000000000..94d7266de --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/math_operations.tsv @@ -0,0 +1,8 @@ ++ plussa +- minus +/ juohkit +÷ juohkit +: juohkit +× geardde +* geardde +· geardde \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/measure/__init__.py b/nemo_text_processing/text_normalization/se/data/measure/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/measure/greek_lower.tsv b/nemo_text_processing/text_normalization/se/data/measure/greek_lower.tsv new file mode 100644 index 000000000..c62630069 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/measure/greek_lower.tsv @@ -0,0 +1,25 @@ +α alfa +β beta +γ gamma +δ delta +ε epsilon +ζ zeta +η eta +θ theta +ι iota +κ kappa +λ lambda +μ mu +ν nu +ξ xi +ο omicron +π pi +ρ rho +σ sigma +τ tau +υ upsilon +φ phi +χ chi +ψ psi +ω omega + diff --git a/nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv b/nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv new file mode 100644 index 000000000..65a5dc041 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv @@ -0,0 +1,13 @@ +cm² kvadráhttasentimehter +cm2 kvadráhttasentimehter +cm sentimehter +mm² kvadráhttamillimehter +mm2 kvadráhttamillimehter +mm millimehter +cm³ guđahassentimehter +cm3 guđahassentimehter +mm³ guđahasmillimehter +mm3 guđahasmillimehter +kg kilográmma +g. grámma +t. tonna diff --git a/nemo_text_processing/text_normalization/se/data/money/__init__.py b/nemo_text_processing/text_normalization/se/data/money/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/se/data/money/currency_major.tsv new file mode 100644 index 000000000..14ff303f2 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/money/currency_major.tsv @@ -0,0 +1,10 @@ +sek ruoŧa ruvdno +nok norgga ruvdno +kr ruvdno +£ punda +eur euro +€ euro +₽ ruppal +$ dollár +czk čeahka ruvdno +kč čeahka ruvdno diff --git a/nemo_text_processing/text_normalization/se/data/money/currency_major_case.tsv b/nemo_text_processing/text_normalization/se/data/money/currency_major_case.tsv new file mode 100644 index 000000000..be0004a44 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/money/currency_major_case.tsv @@ -0,0 +1,9 @@ +$:a dollára +$:t dollárat +$:i dollárii +$:id dolláriid +$:ida dolláriidda +$:iguin dolláriiguin +$:in dolláriin +$:n dollárin +$:s dolláris diff --git a/nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv b/nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv new file mode 100644 index 000000000..d0a0d8c56 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv @@ -0,0 +1,10 @@ +sek ruoŧa ruvnno +nok norgga ruvnno +kr ruvnno +£ punda +eur euro +€ euro +₽ ruppala +$ dollára +czk čeahka ruvnno +kč čeahka ruvnno diff --git a/nemo_text_processing/text_normalization/se/data/money/currency_minor.tsv b/nemo_text_processing/text_normalization/se/data/money/currency_minor.tsv new file mode 100644 index 000000000..e0570256a --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/money/currency_minor.tsv @@ -0,0 +1,10 @@ +sek evre +nok evre +kr evre +£ benne +eur sente +€ sente +₽ +$ sente +czk +kč diff --git a/nemo_text_processing/text_normalization/se/data/money/currency_minor_gen.tsv b/nemo_text_processing/text_normalization/se/data/money/currency_minor_gen.tsv new file mode 100644 index 000000000..82b2e33a3 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/money/currency_minor_gen.tsv @@ -0,0 +1,10 @@ +sek evrre +nok evrre +kr evrre +£ benne +eur sentte +€ sentte +₽ +$ sentte +czk +kč diff --git a/nemo_text_processing/text_normalization/se/data/numbers/__init__.py b/nemo_text_processing/text_normalization/se/data/numbers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/numbers/case_cuodi.tsv b/nemo_text_processing/text_normalization/se/data/numbers/case_cuodi.tsv new file mode 100644 index 000000000..d0d68bd22 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/case_cuodi.tsv @@ -0,0 +1,11 @@ +ess čuođin +com_pl čuđiiguin +com_sg čuđiin +gen_sg čuođe +gen_pl čuđiid +ill_pl čuđiide +ill_sg čuohtái +loc_sg čuođis +nom_pl čuođit +nom_sg čuođi +acc_sg čuođi diff --git a/nemo_text_processing/text_normalization/se/data/numbers/case_duhat.tsv b/nemo_text_processing/text_normalization/se/data/numbers/case_duhat.tsv new file mode 100644 index 000000000..c0820df7e --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/case_duhat.tsv @@ -0,0 +1,11 @@ +ess duháhin +com_pl duháhiiguin +com_sg duháhiin +gen_sg duháha +gen_pl duháhiid +ill_pl duháhiidda +ill_sg duháhii +loc_sg duháhis +nom_pl duháhat +nom_sg duhát +acc_sg duhát diff --git a/nemo_text_processing/text_normalization/se/data/numbers/case_logi.tsv b/nemo_text_processing/text_normalization/se/data/numbers/case_logi.tsv new file mode 100644 index 000000000..ea1b738b0 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/case_logi.tsv @@ -0,0 +1,10 @@ +ess lohkin +com_pl logiiguin +com_sg logiin +gen_sg logi +gen_pl logiid +ill_pl logiide +ill_sg lohkái +loc_sg logis +nom_pl logit +nom_sg logi diff --git a/nemo_text_processing/text_normalization/se/data/numbers/case_miljon.tsv b/nemo_text_processing/text_normalization/se/data/numbers/case_miljon.tsv new file mode 100644 index 000000000..bbacb207f --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/case_miljon.tsv @@ -0,0 +1,11 @@ +ess miljovdnan +com_pl miljovnnaiguin +com_sg miljovnnain +gen_sg miljovnna +gen_pl miljovnnaid +ill_pl miljovnnaide +ill_sg miljovdnii +loc_sg miljovnnas +nom_pl miljovnnat +nom_sg miljovdna +acc_sg miljovnna diff --git a/nemo_text_processing/text_normalization/se/data/numbers/case_nolla.tsv b/nemo_text_processing/text_normalization/se/data/numbers/case_nolla.tsv new file mode 100644 index 000000000..a47f4a4d4 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/case_nolla.tsv @@ -0,0 +1,11 @@ +ess nollan +com_pl nollaiguin +com_sg nollain +gen_sg nolla +gen_pl nollaid +ill_pl nollaide +ill_sg nollii +loc_sg nollas +nom_pl nollat +nom_sg nolla +acc_sg nolla diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit.tsv new file mode 100644 index 000000000..d37226818 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +okta 1 +guokte 2 +golbma 3 +njeallje 4 +vihtta 5 +guhtta 6 +čieža 7 +gávcci 8 +ovcci 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_pl.tsv new file mode 100644 index 000000000..0bdce380f --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_pl.tsv @@ -0,0 +1,9 @@ +ovttaiguin 1 +guvttiiguin 2 +golmmaiguin 3 +njeljiiguin 4 +viđaiguin 5 +guđaiguin 6 +čiežaiguin 7 +gávcciiguin 8 +ovcciiguin 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_sg.tsv new file mode 100644 index 000000000..2d6120b45 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_com_sg.tsv @@ -0,0 +1,9 @@ +ovttain 1 +guvttiin 2 +golmmain 3 +njeljiin 4 +viđain 5 +guđain 6 +čiežain 7 +gávcciin 8 +ovcciin 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ess.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ess.tsv new file mode 100644 index 000000000..cfa9eb785 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ess.tsv @@ -0,0 +1,9 @@ +oktan 1 +guoktin 2 +golbman 3 +njealljin 4 +vihttan 5 +guhttan 6 +čiehčan 7 +gákcin 8 +okcin 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_pl.tsv new file mode 100644 index 000000000..10c282658 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_pl.tsv @@ -0,0 +1,9 @@ +ovttaid 1 +guvttiid 2 +golmmaid 3 +njeljiid 4 +viđaid 5 +guđaid 6 +čiežaid 7 +gávcciid 8 +ovcciid 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_sg.tsv new file mode 100644 index 000000000..e5107f8ee --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_gen_sg.tsv @@ -0,0 +1,9 @@ +ovtta 1 +guovtti 2 +golmma 3 +njealji 4 +viđa 5 +guđa 6 +čieža 7 +gávcci 8 +ovcci 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_pl.tsv new file mode 100644 index 000000000..afdfaf8fc --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_pl.tsv @@ -0,0 +1,9 @@ +ovttaide 1 +guvttiide 2 +golmmaide 3 +njeljiide 4 +viđaide 5 +guđaide 6 +čiežaide 7 +gávcciide 8 +ovcciide 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_sg.tsv new file mode 100644 index 000000000..398e6f028 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_ill_sg.tsv @@ -0,0 +1,9 @@ +oktii 1 +guoktái 2 +golbmii 3 +njealljái 4 +vihttii 5 +guhttii 6 +čižii 7 +gákcái 8 +okcái 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_loc_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_loc_sg.tsv new file mode 100644 index 000000000..7d5c940a0 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_loc_sg.tsv @@ -0,0 +1,9 @@ +ovttas 1 +guovttis 2 +golmmas 3 +njealjis 4 +viđas 5 +guđas 6 +čiežas 7 +gávccis 8 +ovccis 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_nom_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_nom_pl.tsv new file mode 100644 index 000000000..1ef67125b --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_bare_nom_pl.tsv @@ -0,0 +1,9 @@ +ovttat 1 +guovttit 2 +golmmat 3 +njealjit 4 +viđat 5 +guđat 6 +čiežat 7 +gávccit 8 +ovccit 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_case_abbr_suffix.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_case_abbr_suffix.tsv new file mode 100644 index 000000000..11c7195fe --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_case_abbr_suffix.tsv @@ -0,0 +1,9 @@ +ess :n +com_pl :iguin +com_sg :in +gen_sg :a +gen_pl :id +ill_pl :ide +ill_sg :i +loc_sg :s +nom_pl :t diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_com_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_com_pl.tsv new file mode 100644 index 000000000..4307a3a82 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_com_pl.tsv @@ -0,0 +1,9 @@ +okta 1 +guokte 2 +golbma 3 +njeallje 4 +vihtta 5 +guhtta 6 +čieža 7 +gávcci 8 +ovcci 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_com_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_com_sg.tsv new file mode 100644 index 000000000..21c8d3dbc --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_com_sg.tsv @@ -0,0 +1,9 @@ +ovttain 1 +guvttiin 2 +golmmain 3 +njeljiin 4 +viđain 5 +guđain 6 +čiežain 7 +gávcciin 8 +ovcciin 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_ess.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_ess.tsv new file mode 100644 index 000000000..4307a3a82 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_ess.tsv @@ -0,0 +1,9 @@ +okta 1 +guokte 2 +golbma 3 +njeallje 4 +vihtta 5 +guhtta 6 +čieža 7 +gávcci 8 +ovcci 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_gen_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_gen_pl.tsv new file mode 100644 index 000000000..63fea0bdc --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_gen_pl.tsv @@ -0,0 +1,9 @@ +okta 1 +guokte 2 +golbma 3 +njeallje 4 +viđaid 5 +guđaid 6 +čiežaid 7 +gávcciid 8 +ovcciid 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_gen_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_gen_sg.tsv new file mode 100644 index 000000000..f17cf4498 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_gen_sg.tsv @@ -0,0 +1,9 @@ +ovtta 1 +guovtte 2 +golmma 3 +njealje 4 +viđa 5 +guđa 6 +čieža 7 +gávcci 8 +ovcci 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_ill_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_ill_pl.tsv new file mode 100644 index 000000000..4307a3a82 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_ill_pl.tsv @@ -0,0 +1,9 @@ +okta 1 +guokte 2 +golbma 3 +njeallje 4 +vihtta 5 +guhtta 6 +čieža 7 +gávcci 8 +ovcci 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_ill_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_ill_sg.tsv new file mode 100644 index 000000000..f17cf4498 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_ill_sg.tsv @@ -0,0 +1,9 @@ +ovtta 1 +guovtte 2 +golmma 3 +njealje 4 +viđa 5 +guđa 6 +čieža 7 +gávcci 8 +ovcci 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_loc_sg.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_loc_sg.tsv new file mode 100644 index 000000000..f17cf4498 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_loc_sg.tsv @@ -0,0 +1,9 @@ +ovtta 1 +guovtte 2 +golmma 3 +njealje 4 +viđa 5 +guđa 6 +čieža 7 +gávcci 8 +ovcci 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/digit_nom_pl.tsv b/nemo_text_processing/text_normalization/se/data/numbers/digit_nom_pl.tsv new file mode 100644 index 000000000..a48076a18 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/digit_nom_pl.tsv @@ -0,0 +1,9 @@ +okta 1 +guokte 2 +golbma 3 +njeallje 4 +viđat 5 +guđat 6 +čiežat 7 +gávccit 8 +ovccit 9 diff --git a/nemo_text_processing/text_normalization/se/data/numbers/millions.tsv b/nemo_text_processing/text_normalization/se/data/numbers/millions.tsv new file mode 100644 index 000000000..4c6a59549 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/millions.tsv @@ -0,0 +1,11 @@ +milj. miljovdna +miljon miljovdna +milliuvdna miljovdna +millijovdna +miljovdna miljovdna +biljon biljovdna +biljovdna +biljárda +triljon triljovdna +triljovdna +triljárda diff --git a/nemo_text_processing/text_normalization/se/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/se/data/numbers/zero.tsv new file mode 100644 index 000000000..05983b66b --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/zero.tsv @@ -0,0 +1 @@ +nolla 0 diff --git a/nemo_text_processing/text_normalization/se/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/se/data/ordinal/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/ordinal/digit.tsv b/nemo_text_processing/text_normalization/se/data/ordinal/digit.tsv new file mode 100644 index 000000000..d7dd0b90a --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/ordinal/digit.tsv @@ -0,0 +1,9 @@ +vuosttaš 1 +nubbi 2 +goalmmát 3 +njealját 4 +viđat 5 +guđat 6 +čihččet 7 +gávccát 8 +ovccát 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/se/data/ordinal/superscript_digits.tsv b/nemo_text_processing/text_normalization/se/data/ordinal/superscript_digits.tsv new file mode 100644 index 000000000..4a952fd3e --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/ordinal/superscript_digits.tsv @@ -0,0 +1,10 @@ +⁰ 0 +¹ 1 +² 2 +³ 3 +⁴ 4 +⁵ 5 +⁶ 6 +⁷ 7 +⁸ 8 +⁹ 9 diff --git a/nemo_text_processing/text_normalization/se/data/symbols.tsv b/nemo_text_processing/text_normalization/se/data/symbols.tsv new file mode 100644 index 000000000..7b22cb450 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/symbols.tsv @@ -0,0 +1,21 @@ +. čuokkis +: duppalčuokkis +- goallossáhcu +! čuorvvasmearka +; beallečuokkis +' apostrofa +/ finjusáhcu +? gažaldatmearka +, rihkku +_ vuollesárggis +# fáddágilkor +$ dollár +% proseanta +& ja +* asterisk ++ plussa += ovttamađodatmearka +^ sirkumfleaksa +{ gurut bárroruohtu +} olgeš bárroruohtu +~ tilde diff --git a/nemo_text_processing/text_normalization/se/data/telephone/__init__.py b/nemo_text_processing/text_normalization/se/data/telephone/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/telephone/ip_prompt.tsv b/nemo_text_processing/text_normalization/se/data/telephone/ip_prompt.tsv new file mode 100644 index 000000000..21b5dc9e9 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/telephone/ip_prompt.tsv @@ -0,0 +1,6 @@ +IP-čujuhus +IP-čujuhus lea +IP-čujuhusas +IP čujuhus +IP čujuhus lea +IP čujuhusas diff --git a/nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr.tsv b/nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr.tsv new file mode 100644 index 000000000..ea23614e2 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr.tsv @@ -0,0 +1,4 @@ +tel telefonnummár +tel. telefonnummár +tlf telefonnummár +tlf. telefonnummár diff --git a/nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr_nd.tsv b/nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr_nd.tsv new file mode 100644 index 000000000..2cefc3017 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/telephone/telephone_abbr_nd.tsv @@ -0,0 +1,8 @@ +tel telefonnummár +tel. telefonnummár +tlf telefonnummár +tlf. telefonnummár +tel telefovdna +tel. telefovdna +tlf telefovdna +tlf. telefovdna diff --git a/nemo_text_processing/text_normalization/se/data/telephone/telephone_prompt.tsv b/nemo_text_processing/text_normalization/se/data/telephone/telephone_prompt.tsv new file mode 100644 index 000000000..9a6b199a1 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/telephone/telephone_prompt.tsv @@ -0,0 +1,9 @@ +telefovnnas +riŋget +telefonnummir +tel-nummar +telefonmolssa +telefovdnanummár +váldotelefovdnanummárii +telefovdnanummira +telefovdnii diff --git a/nemo_text_processing/text_normalization/se/data/time/__init__.py b/nemo_text_processing/text_normalization/se/data/time/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/data/time/prefix.tsv b/nemo_text_processing/text_normalization/se/data/time/prefix.tsv new file mode 100644 index 000000000..730ffb97a --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/time/prefix.tsv @@ -0,0 +1,8 @@ +kl. klokken +dii. diibmu +dmu. diibmu +dmu diibmu +dii diibmu +diibmu +klokken +klokka diff --git a/nemo_text_processing/text_normalization/se/data/time/suffix.tsv b/nemo_text_processing/text_normalization/se/data/time/suffix.tsv new file mode 100644 index 000000000..19e609109 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/time/suffix.tsv @@ -0,0 +1,2 @@ +ib iđitbeaivet +eb eahketbeaivet diff --git a/nemo_text_processing/text_normalization/se/data/time/time_zone.tsv b/nemo_text_processing/text_normalization/se/data/time/time_zone.tsv new file mode 100644 index 000000000..3e0ade467 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/time/time_zone.tsv @@ -0,0 +1,7 @@ +cst c s t +cet c e t +pst p s t +est e s t +pt p t +et e t +gmt g m t diff --git a/nemo_text_processing/text_normalization/se/data/whitelist.tsv b/nemo_text_processing/text_normalization/se/data/whitelist.tsv new file mode 100644 index 000000000..d2e304dcd --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/whitelist.tsv @@ -0,0 +1,7 @@ +tel. telefonnummár +EU Eurohpauniovdna +EU:a Eurohpauniovnna +EU:i Eurohpauniovdnii +EU:s Eurohpauniovnnas +EU:in Eurohpauniovnnain +EU:n Eurohpauniovdnan diff --git a/nemo_text_processing/text_normalization/se/graph_utils.py b/nemo_text_processing/text_normalization/se/graph_utils.py new file mode 100644 index 000000000..5c5dbe82a --- /dev/null +++ b/nemo_text_processing/text_normalization/se/graph_utils.py @@ -0,0 +1,40 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import delete_space +from pynini.lib import byte, pynutil + +_ALPHA_UPPER = "АÁBCČDĐEFGHIJKLMNŊOPRSŠTŦUVZŽÆØÅÄÖ" +_ALPHA_LOWER = "аábcčdđefghijklmnŋoprsštŧuvzžæøåäö" + +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(_ALPHA_UPPER, _ALPHA_LOWER)]) +TO_UPPER = pynini.invert(TO_LOWER) + +SE_LOWER = pynini.union(*_ALPHA_LOWER).optimize() +SE_UPPER = pynini.union(*_ALPHA_UPPER).optimize() +SE_ALPHA = pynini.union(SE_LOWER, SE_UPPER).optimize() +SE_ALNUM = pynini.union(byte.DIGIT, SE_ALPHA).optimize() + +bos_or_space = pynini.union("[BOS]", " ") +eos_or_space = pynini.union("[EOS]", " ") + +ensure_space = pynini.cross(pynini.closure(delete_space, 0, 1), " ") + + +def make_spacer(deterministic=True): + spacer = pynini.accep("") + if not deterministic: + spacer |= pynutil.insert(" ") + return spacer diff --git a/nemo_text_processing/text_normalization/se/taggers/__init__.py b/nemo_text_processing/text_normalization/se/taggers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/taggers/abbreviation.py b/nemo_text_processing/text_normalization/se/taggers/abbreviation.py new file mode 100644 index 000000000..a8b624543 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/abbreviation.py @@ -0,0 +1,52 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.se.graph_utils import SE_UPPER +from pynini.lib import pynutil + + +class AbbreviationFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, etc. + e.g. "ABC" -> tokens { abbreviation { value: "A B C" } } + + Args: + whitelist: whitelist FST + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, whitelist: 'pynini.FstLike', deterministic: bool = True): + super().__init__(name="abbreviation", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + # A.B.C. -> A. B. C. + graph = SE_UPPER + dot + pynini.closure(insert_space + SE_UPPER + dot, 1) + # A.B.C. -> A.B.C. + graph |= SE_UPPER + dot + pynini.closure(SE_UPPER + dot, 1) + # ABC -> A B C + graph |= SE_UPPER + pynini.closure(insert_space + SE_UPPER, 1) + + # exclude words that are included in the whitelist + if whitelist is not None: + graph = pynini.compose( + pynini.difference(pynini.project(graph, "input"), pynini.project(whitelist.graph, "input")), graph + ) + + graph = pynutil.insert("value: \"") + graph.optimize() + pynutil.insert("\"") + graph = self.add_tokens(graph) + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/cardinal.py b/nemo_text_processing/text_normalization/se/taggers/cardinal.py new file mode 100644 index 000000000..684f1fa21 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/cardinal.py @@ -0,0 +1,434 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, +) +from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA, make_spacer +from nemo_text_processing.text_normalization.se.utils import CASE_KEYS, get_abs_path, load_case_forms, load_labels +from pynini.lib import pynutil + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by space) + and converts to a string of digits: + "1 000" -> "1000" + Args: + fst: Any pynini.FstLike object. Function composes fst onto string parser fst + + Returns: + fst: A pynini.FstLike object + """ + exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string + + cardinal_separator = pynini.union(NEMO_SPACE, ".") + cardinal_string = pynini.closure( + NEMO_DIGIT, 1 + ) # For string w/o punctuation (used for page numbers, thousand series) + + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + exactly_three_digits + ) + + return cardinal_string @ fst + + +def load_cased_digits(bare=True): + digits_cased = {} + for key in CASE_KEYS: + if bare: + fkey = f"bare_{key}" + else: + fkey = key + digits_cased[key] = {} + for label in load_labels(get_abs_path(f"data/numbers/digit_{fkey}.tsv")): + digits_cased[key][label[1]] = label[0] + digits_cased["nom_sg"] = {} + for label in load_labels(get_abs_path(f"data/numbers/digit.tsv")): + digits_cased["nom_sg"][label[1]] = label[0] + return digits_cased + + +def build_cased_number_fsts(deterministic=True): + """ + Builds case/number forms for numerals + See: https://oahpa.no/sme/gramm/logut.eng.html + for teens and tens; for longer numbers Nickel and Sammallahti (2011) say + 'i lengre tallord bøyes bare enere' ('in longer number words only ones are inflected') + """ + digits_nom = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) + digits_nom_no_one = (NEMO_DIGIT - "1") @ digits_nom + cuodi_cased = load_case_forms(get_abs_path("data/numbers/case_cuodi.tsv"), True) + logi_cased = load_case_forms(get_abs_path("data/numbers/case_logi.tsv"), True) + duhat_cased = load_case_forms(get_abs_path("data/numbers/case_duhat.tsv"), True) + nolla_cased = load_case_forms(get_abs_path("data/numbers/case_nolla.tsv"), True) + endings_cased = load_case_forms(get_abs_path("data/numbers/digit_case_abbr_suffix.tsv")) + spacer = make_spacer(deterministic) + + if not deterministic: + digits_nom |= pynini.cross("1", "akta") + + # digits + def get_digit_cased_fst(bare=True, deterministic=deterministic): + digits_cased = load_cased_digits(bare) + + digits_cased_fst = {} + for k in digits_cased: + digits_cased_fst[k] = pynini.string_map((k, v) for k, v in digits_cased[k].items()) + if not deterministic: + if k == "nom_sg": + digits_cased_fst[k] |= pynini.cross("1", "akta") + elif k == "gen_sg" and bare: + digits_cased_fst[k] |= pynini.cross("2", "guovtti") + return digits_cased_fst + + digits_bare_cased_fst = get_digit_cased_fst(bare=True) + digits_cased_fst = get_digit_cased_fst(bare=False) + + # for hundreds, thousands, etc. + digits_nom_prefix = (NEMO_DIGIT - "1") @ digits_nom + digits_nom_prefix |= pynutil.delete("1") + if not deterministic: + digits_nom_prefix |= pynini.cross("1", "okta") + digits_nom_prefix |= pynini.cross("1", "akta") + + digits_cased_fst_pfx = {} + digits_cased_fst_pfx["nom_sg"] = digits_nom_prefix + for k in digits_cased_fst: + if k not in digits_cased_fst_pfx: + digits_cased_fst_pfx[k] = digits_cased_fst[k] + + # zero + nolla_cased_fst = {} + for k in nolla_cased: + nolla_cased_fst[k] = pynini.cross("0", nolla_cased[k]) + + # teens + teens_cased_fst = {} + nuppelogin = pynutil.insert("nuppelogin") + for k in digits_cased_fst: + assert "nom_sg" in digits_cased_fst + assert "nom_sg" in logi_cased + if k == "nom_sg": + teens_cased_fst[k] = pynutil.delete("1") + digits_cased_fst[k] + pynutil.insert("nuppelohkái") + else: + teens_cased_fst[k] = pynutil.delete("1") + digits_cased_fst[k] + pynutil.insert(f"nuppe{logi_cased[k]}") + if not deterministic: + if k in ["nom_pl", "gen_pl", "acc_pl", "loc_pl"]: + dbc = digits_bare_cased_fst[k] + teens_cased_fst[k] = pynutil.delete("1") + dbc + pynutil.insert(f"nuppe{logi_cased[k]}") + if k == "ess": + teens_cased_fst["ess"] |= pynutil.delete("1") + digits_cased_fst["ess"] + nuppelogin + teens_cased_fst["ess"] |= pynutil.delete("1") + digits_cased_fst["nom_pl"] + nuppelogin + + # tens + tens_cased_fst = {} + # com.sg/loc.pl is different for 'logi' + for k in digits_cased_fst: + logi = "logi" + digit_cased_no_one = (NEMO_DIGIT - "1") @ digits_bare_cased_fst[k] + digit_nom_no_one = (NEMO_DIGIT - "1") @ digits_bare_cased_fst["nom_sg"] + if k == 'com_sg': + logi = "logiin" + ten = digit_cased_no_one + else: + ten = digits_nom_no_one + # 20 -> guvttiin/logiin + tens_cased_fst[k] = digit_cased_no_one + spacer + pynini.cross("0", logi_cased[k]) + tens_cased_fst[k] |= pynini.cross("10", logi_cased[k]) + # e.g.: https://gtweb.uit.no/cgi-bin/smi/smi.cgi?text=vihttalogi&pos=Any&mode=full&lang=sme&plang=eng&action=paradigm + if not deterministic: + if k in ["nom_pl", "gen_pl", "loc_pl", "ess"]: + tens_cased_fst[k] |= digit_nom_no_one + spacer + pynini.cross("0", logi_cased[k]) + if k == "ess": + digit_nom_pl = (NEMO_DIGIT - "1") @ digits_cased_fst["nom_pl"] + tens_cased_fst[k] |= digit_nom_pl + spacer + pynini.cross("0", "login") + + tens_cased_fst[k] |= digit_nom_no_one + spacer + pynini.cross("0", "login") + if k == "nom_sg": + tens_cased_fst[k] |= digit_nom_no_one + spacer + pynini.cross("0", "lohki") + if k == "com_pl": + digit_gen_pl = (NEMO_DIGIT - "1") @ digits_cased_fst["gen_pl"] + tens_cased_fst[k] |= digit_gen_pl + spacer + pynini.cross("0", logi_cased[k]) + # 23 -> guvttiin/logiin/golmmain + tens_cased_fst[k] |= ten + spacer + pynutil.insert(logi) + spacer + digits_bare_cased_fst[k] + + # two digits + two_digit_cased_fsts = {} + two_digit_cased_fsts_sfx = {} + two_digits_fst = None + for k in digits_cased_fst: + two_digit_cased_fsts[k] = ( + tens_cased_fst[k] | teens_cased_fst[k] | (pynutil.delete("0") + digits_bare_cased_fst[k]) + ) + if k != "nom_sg": + two_digit_cased_fsts_sfx[k] = two_digit_cased_fsts[k] + pynutil.delete(endings_cased[k]) + if two_digits_fst is None: + two_digits_fst = two_digit_cased_fsts_sfx[k] + else: + two_digits_fst |= two_digit_cased_fsts_sfx[k] + + digits_prefix_cased_fst = {} + for k in digits_cased_fst: + if k in ["ess", "ill_pl", "nom_pl", "gen_pl", "com_pl"]: + digits_prefix_cased_fst[k] = (NEMO_DIGIT - "1") @ digits_bare_cased_fst[k] + else: + digits_prefix_cased_fst[k] = (NEMO_DIGIT - "1") @ digits_cased_fst[k] + if not deterministic: + digits_prefix_cased_fst[k] |= "1" @ digits_cased_fst[k] + if k == "com_pl": + digits_prefix_cased_fst[k] |= (NEMO_DIGIT - "1") @ digits_cased_fst["gen_pl"] + if k == "nom_sg": + digits_prefix_cased_fst[k] |= pynini.cross("1", "akta") + + # bare hundreds + bare_hundreds_fst = {} + for k in digits_cased_fst: + bare_hundred = pynini.cross("00", cuodi_cased[k]) + if k == "gen_sg": + bare_hundred = pynini.cross("00", "čuođi") + prefix_digit = (NEMO_DIGIT - "1") @ digits_prefix_cased_fst[k] + prefix_digit |= pynutil.delete("1") + if not deterministic and k == "ess": + bare_hundred |= pynini.cross("00", "čuohtin") + bare_hundreds_fst[k] = prefix_digit + spacer + bare_hundred + + def select_tens(tens_cased): + return ((NEMO_DIGIT - "0") + pynini.accep("0")) @ tens_cased + + # 3 digit + prefix_hundreds = digits_nom_prefix + pynutil.insert("čuođi") + just_tens_nom = ((NEMO_DIGIT - "1" - "0") + pynutil.insert("0")) @ tens_cased_fst['nom_sg'] + hundreds_fst = {} + for k in digits_cased_fst: + hundreds_fst[k] = prefix_hundreds + pynutil.delete("0") + spacer + digits_bare_cased_fst[k] + hundreds_fst[k] |= prefix_hundreds + spacer + teens_cased_fst[k] + if k in ["loc_pl", "com_sg"]: + hundreds_fst[k] |= prefix_hundreds + spacer + tens_cased_fst[k] + if not deterministic: + hundreds_fst[k] |= prefix_hundreds + spacer + just_tens_nom + spacer + digits_bare_cased_fst[k] + hundreds_fst[k] |= prefix_hundreds + spacer + select_tens(tens_cased_fst[k]) + else: + hundreds_fst[k] |= prefix_hundreds + spacer + just_tens_nom + spacer + digits_bare_cased_fst[k] + hundreds_fst[k] |= prefix_hundreds + spacer + select_tens(tens_cased_fst[k]) + if not deterministic: + hundreds_fst[k] |= prefix_hundreds + spacer + tens_cased_fst[k] + + three_digit_cased_fsts = {} + for k in two_digit_cased_fsts: + three_digit_cased_fsts[k] = (pynutil.delete("0") + two_digit_cased_fsts[k]) | hundreds_fst[k] + + # thousands + bare_thousands_fst = {} + for k in digits_cased_fst: + bare_thousand = pynini.cross("000", duhat_cased[k]) + prefix_digit = (NEMO_DIGIT - "1") @ digits_prefix_cased_fst[k] + prefix_digit |= pynutil.delete("1") + if not deterministic and k == "sg_gen": + bare_thousand |= pynini.cross("000", "duhát") + bare_thousands_fst[k] = prefix_digit + spacer + bare_thousand + thousands_fst = {} + prefix_thousands = digits_nom_prefix + pynutil.insert("duhát") + for k in digits_cased_fst: + thousands_fst[k] = prefix_thousands + pynutil.delete("00") + spacer + digits_bare_cased_fst[k] + + return { + "tens": tens_cased_fst, + "teens": teens_cased_fst, + "digits": digits_bare_cased_fst, + "zero": nolla_cased_fst, + "two_digit_cased_fsts": two_digit_cased_fsts, + "two_digit_cased_fsts_sfx": two_digit_cased_fsts_sfx, + "two_digit_fst": two_digits_fst, + "bare_hundreds": bare_hundreds_fst, + "hundreds": hundreds_fst, + "three_digit_cased_fsts": three_digit_cased_fsts, + "bare_thousands": bare_thousands_fst, + "thousands": thousands_fst, + } + + +def make_million( + initial: str, at_least_one_non_zero_digit_no_one: 'pynini.FstLike', deterministic=True +) -> 'pynini.FstLike': + spacer = make_spacer(deterministic) + + graph_million = pynutil.add_weight(pynini.cross("001", f"{initial}iljovdna") + spacer, -0.001) + graph_million |= at_least_one_non_zero_digit_no_one + spacer + pynutil.insert(f"{initial}iljovnna") + spacer + if not deterministic: + graph_million |= pynutil.add_weight(pynini.cross("001", f"{initial}iljon") + spacer, -0.001) + graph_million |= pynutil.add_weight(pynini.cross("001", f"{initial}iljun") + spacer, -0.001) + graph_million |= pynutil.add_weight(pynini.cross("001", f"{initial}illiuvdna") + spacer, -0.001) + graph_million |= at_least_one_non_zero_digit_no_one + spacer + pynutil.insert(f"{initial}illiuvnna") + spacer + graph_million |= pynutil.delete("000") + return graph_million + + +def make_milliard( + initial: str, at_least_one_non_zero_digit_no_one: 'pynini.FstLike', deterministic=True +) -> 'pynini.FstLike': + spacer = make_spacer(deterministic) + + graph_milliard = pynutil.add_weight(pynini.cross("001", f"{initial}iljárda"), -0.001) + spacer + graph_milliard |= at_least_one_non_zero_digit_no_one + spacer + pynutil.insert(f"{initial}iljárdda") + spacer + graph_milliard |= pynutil.delete("000") + return graph_milliard + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + "1000" -> cardinal { integer: "duhat" } + "2 000 000" -> cardinal { integer: "guoktemiljovnna" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) + + # Any single digit + graph_digit = digit + self.digit_inverse = pynini.invert(digit) + digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + + graph_zero = zero + if not deterministic: + graph_zero |= pynini.cross("0", "nulla") + graph_digit |= pynini.cross("1", "akta") + + teen = pynutil.delete("1") + digit + pynutil.insert("nuppelohkái") + teen |= pynini.cross("10", "logi") + ties = digits_no_one + pynini.cross("0", "logi") + ties |= digits_no_one + pynutil.insert("logi") + digit + + graph_tens = teen + graph_ties = ties + + self.tens = graph_tens.optimize() + self.ties = graph_ties.optimize() + + two_digit_non_zero = pynini.union(graph_tens, graph_ties, (pynutil.delete("0") + graph_digit)) + graph_two_digit_non_zero = pynini.union(graph_digit, two_digit_non_zero) + + self.two_digit_non_zero = graph_two_digit_non_zero.optimize() + + # Three digit strings + hundreds = digits_no_one + pynutil.insert("čuođi") + hundreds |= pynini.cross("1", "čuođi") + if not deterministic: + hundreds |= pynini.cross("1", "oktačuođi") + hundreds |= pynini.cross("1", "aktačuođi") + + final_hundreds = hundreds + pynini.union(two_digit_non_zero, pynutil.delete("00")) + graph_hundreds = pynini.union(final_hundreds, graph_two_digit_non_zero) + + self.hundreds = graph_hundreds.optimize() + + # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) + graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + (graph_tens | graph_ties)) + + graph_hundreds_component_at_least_one_non_zero_digit = graph_hundreds_component | ( + pynutil.delete("00") + graph_digit + ) + graph_hundreds_component_at_least_one_non_zero_digit_no_one = graph_hundreds_component | ( + pynutil.delete("00") + digits_no_one + ) + self.graph_hundreds_component_at_least_one_non_zero_digit = ( + graph_hundreds_component_at_least_one_non_zero_digit + ) + self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( + graph_hundreds_component_at_least_one_non_zero_digit_no_one.optimize() + ) + + duhat = pynutil.insert("duhát") + duhat_cross = pynini.cross("001", "duhát") + if not deterministic: + duhat_cross |= pynini.cross("001", "duhát ") + + graph_thousands_component_at_least_one_non_zero_digit = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit, + graph_hundreds_component_at_least_one_non_zero_digit_no_one + + duhat + + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + duhat_cross + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + ) + self.graph_thousands_component_at_least_one_non_zero_digit = ( + graph_thousands_component_at_least_one_non_zero_digit + ) + + graph_thousands_component_at_least_one_non_zero_digit_no_one = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit_no_one, + graph_hundreds_component_at_least_one_non_zero_digit_no_one + + duhat + + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + duhat_cross + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + ) + + graph_million = make_million("m", graph_hundreds_component_at_least_one_non_zero_digit_no_one, deterministic) + + graph_milliard = make_milliard("m", graph_hundreds_component_at_least_one_non_zero_digit_no_one, deterministic) + + graph_billion = make_million("b", graph_hundreds_component_at_least_one_non_zero_digit_no_one, deterministic) + + graph_billiard = make_milliard("b", graph_hundreds_component_at_least_one_non_zero_digit_no_one, deterministic) + + graph_trillion = make_million("tr", graph_hundreds_component_at_least_one_non_zero_digit_no_one, deterministic) + + graph_trilliard = make_milliard( + "tr", graph_hundreds_component_at_least_one_non_zero_digit_no_one, deterministic + ) + + self.graph_higher = ( + graph_trilliard + graph_trillion + graph_billiard + graph_billion + graph_milliard + graph_million + ) + graph = self.graph_higher + (graph_thousands_component_at_least_one_non_zero_digit | pynutil.delete("000000")) + + self.graph = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT ** 24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), SE_ALPHA, SE_ALPHA, NEMO_SIGMA + ) + ) + self.graph |= graph_zero + + self.graph = filter_punctuation(self.graph).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/date.py b/nemo_text_processing/text_normalization/se/taggers/date.py new file mode 100644 index 000000000..e6c830381 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/date.py @@ -0,0 +1,126 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.se.graph_utils import TO_LOWER +from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels +from pynini.lib import pynutil + +delete_leading_zero = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + "skábmamánu 8. b. 1956" -> date { month: "skábmamánnu" day: "gávccát" year: "duhátovccičuođivihttalogiguhtta" preserve_order: true } + "8/11/1956" -> date { day: "gávccát" month: "skábmamánnu" year: "duhátovccičuođivihttalogiguhtta" } + "8-11 1956" -> date { day: "gávccát" month: "skábmamánnu" year: "duhátovccičuođivihttalogiguhtta" } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + month_abbr_graph = load_labels(get_abs_path("data/dates/months_abbr.tsv")) + number_to_month = pynini.string_file(get_abs_path("data/dates/numbers.tsv")).optimize() + if not deterministic: + number_to_month |= pynini.cross("1", "ođđajagemánnu") + + month_names = pynini.project(number_to_month, "output") + month_nom_to_gen_map = pynini.string_map([("mánnu", "mánu")]) + self.months_nom2gen = month_names @ pynini.cdrewrite(month_nom_to_gen_map, "", "[EOS]", NEMO_SIGMA) + self.months_gen2nom = pynini.invert(self.months_nom2gen) + self.months_num2gen = (number_to_month @ self.months_nom2gen).optimize() + month_graph = self.months_gen2nom + + month_abbr_graph = pynini.string_map(month_abbr_graph) + month_abbr_graph = ( + pynutil.add_weight(month_abbr_graph, weight=0.0001) + | ((TO_LOWER + pynini.closure(NEMO_CHAR)) @ month_abbr_graph) + ) + pynini.closure(pynutil.delete(".", weight=-0.0001), 0, 1) + + self.month_abbr = month_abbr_graph + month_graph |= (TO_LOWER + pynini.closure(NEMO_CHAR)) @ month_graph + month_graph |= month_abbr_graph + + numbers = cardinal.graph + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + # 01, 31, 1 + digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ ordinal.graph_bare_ordinals + day = (pynutil.insert("day: \"") + digit_day + pynutil.insert("\"")).optimize() + + digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + graph_number_to_month = digit_month @ number_to_month + + month_name = (pynutil.insert("month: \"") + month_graph + pynutil.insert("\"")).optimize() + month_number = (pynutil.insert("month: \"") + graph_number_to_month + pynutil.insert("\"")).optimize() + + # prefer cardinal over year + year = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 1, 3) # 90, 990, 1990 + year @= numbers + self.year = year.optimize() + + year_only = pynutil.insert("year: \"") + year + pynutil.insert("\"") + + beaivi = pynutil.delete(pynini.union("b.", "beaivi")) + preserve_order = pynutil.insert(" preserve_order: true") + + graph_md = month_name + NEMO_SPACE + day + pynini.closure(pynutil.delete("."), 0, 1) + NEMO_SPACE + beaivi + self.md = (graph_md + preserve_order).optimize() + + graph_mdy = graph_md + pynini.closure(pynini.accep(" ") + year_only, 0, 1) + preserve_order + self.mdy = graph_mdy.optimize() + + graph_dmy = ( + day + + pynutil.delete("/") + + insert_space + + month_number + + pynini.closure(pynutil.delete("/") + insert_space + year_only, 0, 1) + ) + self.dmy = graph_dmy.optimize() + graph_ymd = ( + year_only + + pynutil.delete("/") + + insert_space + + month_number + + pynini.closure(pynutil.delete("/") + insert_space + day, 0, 1) + ) + self.ymd = graph_ymd.optimize() + + separators = ["/", "-"] + for sep in separators: + year_optional = pynini.closure(NEMO_SPACE + year_only, 0, 1) + new_graph = day + pynini.cross(sep, " ") + month_number + year_optional + graph_dmy |= new_graph + + final_graph = graph_dmy + # final_graph |= year_only + final_graph |= graph_ymd + final_graph |= graph_mdy + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/decimal.py b/nemo_text_processing/text_normalization/se/taggers/decimal.py new file mode 100644 index 000000000..00b436da0 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/decimal.py @@ -0,0 +1,105 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.se.utils import get_abs_path +from pynini.lib import pynutil + +quantities = pynini.string_file(get_abs_path("data/numbers/millions.tsv")) + + +def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_thousand: 'pynini.FstLike',) -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 1 miljárda -> integer_part: "okta" quantity: "miljárda" + e.g. 1,5 miljárdda -> integer_part: "okta" fractional_part: "vihtta" quantity: "miljárdda" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + nom_to_gen_endings = pynini.string_map(("on", "ovnna"), ("ovdna", "ovnna"), ("árda", "árdda",)) + quantities_gen = quantities @ pynini.cdrewrite(nom_to_gen_endings, "", "[EOS]", NEMO_SIGMA) + + res = ( + pynutil.insert("integer_part: \"") + + cardinal_up_to_thousand + + pynutil.insert("\"") + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert(" quantity: \"") + + quantities_gen + + pynutil.insert("\"") + ) + res |= ( + pynutil.insert("integer_part: \"") + + pynini.cross("1", "okta") + + pynutil.insert("\"") + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert(" quantity: \"") + + quantities + + pynutil.insert("\"") + ) + res |= ( + decimal + + pynini.closure(pynutil.delete(" "), 0, 1) + + pynutil.insert(" quantity: \"") + + quantities_gen + + pynutil.insert("\"") + ) + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -12,5006 biljovnna -> decimal { negative: "true" integer_part: "guoktenuppelohkái" fractional_part: "vihtta nolla nolla guhtta" quantity: "biljovnna" } + 1 miljovdna -> decimal { integer_part: "okta" quantity: "miljovdna" } + + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + cardinal_graph_hundreds_one_non_zero = cardinal.graph_hundreds_component_at_least_one_non_zero_digit_no_one + + self.graph = cardinal.two_or_three_digits_read_frac + + if not deterministic: + self.graph |= cardinal.single_digits_graph.optimize() + self.graph |= cardinal_graph + + point = pynutil.delete(",") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + self.graph_fractional = pynutil.insert("fractional_part: \"") + self.graph + pynutil.insert("\"") + self.graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure(self.graph_integer + pynutil.insert(" "), 0, 1) + + point + + pynutil.insert(" ") + + self.graph_fractional + ) + self.final_graph_wo_sign = final_graph_wo_sign + + quantity_w_abbr = get_quantity(final_graph_wo_sign, cardinal_graph_hundreds_one_non_zero) + self.final_graph_wo_negative = final_graph_wo_sign | quantity_w_abbr + + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/electronic.py b/nemo_text_processing/text_normalization/se/taggers/electronic.py new file mode 100644 index 000000000..528b1838e --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/electronic.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels +from pynini.lib import pynutil + +common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] +symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: email addresses + e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true } + e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + dot = pynini.accep(".") + accepted_common_domains = pynini.union(*common_domains) + accepted_symbols = pynini.union(*symbols) - dot + accepted_characters = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols) + accepted_characters_with_dot = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols | dot) + + # email + username = ( + pynutil.insert("username: \"") + + accepted_characters_with_dot + + pynutil.insert("\"") + + (pynini.cross('@', ' ') | pynini.cross('(at)', ' ')) + ) + domain_graph = accepted_characters + dot + accepted_characters + domain_graph = pynutil.insert("domain: \"") + domain_graph + pynutil.insert("\"") + domain_common_graph = ( + pynutil.insert("domain: \"") + + accepted_characters + + accepted_common_domains + + pynini.closure((accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1) + + pynutil.insert("\"") + ) + graph = (username + domain_graph) | domain_common_graph + + # url + protocol_start = pynini.accep("https://") | pynini.accep("http://") + protocol_end = pynini.cross("www.", "v v v") + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = pynutil.insert("protocol: \"") + protocol + pynutil.insert("\"") + graph |= protocol + insert_space + (domain_graph | domain_common_graph) + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/fraction.py b/nemo_text_processing/text_normalization/se/taggers/fraction.py new file mode 100644 index 000000000..b48f65e54 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/fraction.py @@ -0,0 +1,96 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.se.graph_utils import ensure_space +from nemo_text_processing.text_normalization.se.utils import get_abs_path +from pynini.lib import pynutil + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "23 4/5" -> + tokens { fraction { integer: "tjugotre" numerator: "fyra" denominator: "femtedel" } } + # en åttondel (1/8) + + Args: + cardinal: CardinalFst + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + ordinal_graph = ordinal.graph + numerator_graph = cardinal.graph_en + + fractional_endings = pynini.string_map( + [ + ("ljarte", "ljarddel"), + ("tionde", "tiondel"), + ("tonde", "tondel"), + ("ljonte", "ljondel"), + ("lliarte", "lliarddel"), + ("llionte", "lliondel"), + ("tusende", "tusendel"), + ("te", "tedel"), + ("de", "dedel"), + ("je", "jedel"), + ("drade", "dradel"), + ("a", "adel"), + ] + ) + alt_fractional_endings = pynini.string_map([("tondel", "tondedel"), ("tiondel", "tiondedel")]) + lexicalised = pynini.string_map([("andradel", "halv"), ("fjärdedel", "kvart")]) + alt_lexicalised = pynini.string_map([("halv", "andradel"), ("kvart", "fjärdedel"), ("kvart", "kvarts")]) + + fractions = ( + ordinal_graph + @ pynini.cdrewrite(fractional_endings, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite(lexicalised, "[BOS]", "[EOS]", NEMO_SIGMA) + ) + fractions_alt = ( + fractions + @ pynini.cdrewrite(alt_fractional_endings, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite(alt_lexicalised, "[BOS]", "[EOS]", NEMO_SIGMA) + ) + if not deterministic: + fractions |= fractions_alt + + self.fractions = fractions + + fractional_pl_endings = pynini.string_map([("kvart", "kvartar"), ("halv", "halva"), ("del", "delar")]) + fractions_pl = fractions @ pynini.cdrewrite(fractional_pl_endings, "", "[EOS]", NEMO_SIGMA) + self.fractional_plural_endings = fractional_pl_endings + self.fractions_plural = fractions_pl + + integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + numerator = ( + pynutil.insert("numerator: \"") + numerator_graph + (pynini.cross("/", "\" ") | pynini.cross(" / ", "\" ")) + ) + + denominator = pynutil.insert("denominator: \"") + fractions + pynutil.insert("\"") + + graph = pynini.closure(integer + pynini.accep(" "), 0, 1) + (numerator + denominator) + graph |= pynini.closure(integer + ensure_space, 0, 1) + pynini.compose( + pynini.string_file(get_abs_path("data/numbers/fraction.tsv")), (numerator + denominator) + ) + + self.graph = graph + final_graph = self.add_tokens(self.graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/measure.py b/nemo_text_processing/text_normalization/se/taggers/measure.py new file mode 100644 index 000000000..443d9e889 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/measure.py @@ -0,0 +1,212 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NON_BREAKING_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, + delete_space, + delete_zero_or_one_space, +) +from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA, TO_LOWER +from nemo_text_processing.text_normalization.se.utils import get_abs_path +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, suppletive aware, e.g. + -12kg -> measure { negative: "true" cardinal { integer: "guoktenuppelohkái" } units: "kilográmma" } + 1kg -> measure { cardinal { integer: "okta" } units: "kilográmma" } + ,5kg -> measure { decimal { fractional_part: "guhtta" } units: "kilográmma" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + unit_simple = pynini.string_file(get_abs_path("data/measure/unit_simple.tsv")) + simple_endings = pynini.string_file(get_abs_path("data/inflection/simple.tsv")) + graph_simple = unit_simple | unit_simple + simple_endings + + graph_plurals = pynini.string_file(get_abs_path("data/measure/unit_plural.tsv")) + greek_lower = pynini.string_file(get_abs_path("data/measure/greek_lower.tsv")) + greek_upper = pynutil.insert("stuorra ") + pynini.string_file(get_abs_path("data/measure/greek_upper.tsv")) + greek = greek_lower | greek_upper + + graph_unit |= pynini.compose( + pynini.closure(TO_LOWER, 1) + (SE_ALPHA | TO_LOWER) + pynini.closure(SE_ALPHA | TO_LOWER), graph_unit + ).optimize() + + graph_unit_plural = convert_space(graph_unit @ graph_plurals) + graph_unit = convert_space(graph_unit) + # placeholder: probably needs to be a file + graph_unit_gen = graph_unit + + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + graph_unit2 = ( + pynutil.delete("/") + + delete_zero_or_one_space + + pynutil.insert(NEMO_NON_BREAKING_SPACE) + + graph_unit_gen + + pynutil.insert(" mielde") + ) + + optional_graph_unit2 = pynini.closure( + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, 0, 1, + ) + + unit_plural = ( + pynutil.insert("units: \"") + + (graph_unit_plural + optional_graph_unit2 | graph_unit2) + + pynutil.insert("\"") + ) + + unit_singular = ( + pynutil.insert("units: \"") + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert("\"") + ) + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal.final_graph_wo_negative_en + + delete_space + + pynutil.insert(" } ") + + unit_plural + ) + + # support radio FM/AM + subgraph_decimal |= ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + delete_space + + pynutil.insert(" } ") + + pynutil.insert("units: \"") + + pynini.union("AM", "FM") + + pynutil.insert("\"") + ) + + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + (NEMO_SIGMA @ cardinal_graph) + + delete_space + + pynutil.insert("\"") + + pynutil.insert(" } ") + + unit_plural + ) + self.subgraph_cardinal = subgraph_cardinal + + unit_graph = ( + pynutil.insert("cardinal { integer: \"-\" } units: \"") + + ((pynini.cross("/", "per") + delete_zero_or_one_space) | (pynini.accep("per") + pynutil.delete(" "))) + + pynutil.insert(NEMO_NON_BREAKING_SPACE) + + graph_unit + + pynutil.insert("\" preserve_order: true") + ) + + decimal_dash_alpha = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynini.cross('-', '') + + pynutil.insert(" } units: \"") + + pynini.closure(SE_ALPHA, 1) + + pynutil.insert("\"") + ) + + decimal_times = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } units: \"") + + (pynini.cross(pynini.union('x', "X"), 'x') | pynini.cross(pynini.union('x', "X"), ' geardde')) + + pynutil.insert("\"") + ) + + alpha_dash_decimal = ( + pynutil.insert("units: \"") + + pynini.closure(SE_ALPHA, 1) + + pynini.accep('-') + + pynutil.insert("\"") + + pynutil.insert(" decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" } preserve_order: true") + ) + + subgraph_fraction = ( + pynutil.insert("fraction { ") + fraction.graph + delete_space + pynutil.insert(" } ") + unit_plural + ) + + math_operations = pynini.string_file(get_abs_path("data/math_operations.tsv")) + delimiter = pynini.accep(" ") | pynutil.insert(" ") + + equals = pynini.cross("=", "lea") + + math = ( + (cardinal_graph | SE_ALPHA | greek) + + delimiter + + math_operations + + (delimiter | SE_ALPHA) + + cardinal_graph + + delimiter + + equals + + delimiter + + (cardinal_graph | SE_ALPHA | greek) + ) + + math = ( + pynutil.insert("units: \"math\" cardinal { integer: \"") + + math + + pynutil.insert("\" } preserve_order: true") + ) + final_graph = ( + subgraph_decimal + | subgraph_cardinal + | unit_graph + | decimal_dash_alpha + | decimal_times + | alpha_dash_decimal + | subgraph_fraction + | math + ) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def get_range(self, cardinal: GraphFst): + """ + Returns range forms for measure tagger, e.g. 2-3, 2x3, 2*2 + + Args: + cardinal: cardinal GraphFst + """ + range_graph = cardinal + pynini.cross(pynini.union("-", " - "), " till ") + cardinal + + for x in [" x ", "x"]: + range_graph |= cardinal + pynini.cross(x, " gånger ") + cardinal + + for x in ["*", " * "]: + range_graph |= cardinal + pynini.cross(x, " gånger ") + cardinal + return range_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/money.py b/nemo_text_processing/text_normalization/se/taggers/money.py new file mode 100644 index 000000000..64fc461bc --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/money.py @@ -0,0 +1,232 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA, ensure_space +from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels +from pynini.lib import pynutil + +min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) +min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) +maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +maj_singular_nt = pynini.string_file((get_abs_path("data/money/currency_major_nt.tsv"))) +maj_plural = pynini.string_file((get_abs_path("data/money/currency_plurals.tsv"))) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, suppletive aware, e.g. + $12,05 -> money { integer_part: "tolv" currency_maj: "dollar" fractional_part: "fem" currency_min: "cent" preserve_order: true } + $12,0500 -> money { integer_part: "tolv" currency_maj: "dollar" fractional_part: "fem" currency_min: "cent" preserve_order: true } + $1 -> money { currency_maj: "dollar" integer_part: "en" } + $1,00 -> money { currency_maj: "dollar" integer_part: "en" } + $0,05 -> money { fractional_part: "fem" currency_min: "cent" preserve_order: true } + $1 miljon -> money { currency_maj: "dollar" integer_part: "en" quantity: "miljon" } + $1,2 miljon -> money { currency_maj: "dollar" integer_part: "en" fractional_part: "två" quantity: "miljon" } + $1,2320 -> money { currency_maj: "dollar" integer_part: "en" fractional_part: "two three two" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph_no_one_en + cardinal_graph_ett = cardinal.graph_no_one + graph_decimal_final = decimal.final_graph_wo_negative_w_abbr_en + graph_decimal_final_ett = decimal.final_graph_wo_negative_w_abbr + + maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + maj_singular_labels_nt = load_labels(get_abs_path("data/money/currency_major_nt.tsv")) + maj_unit_plural = convert_space(maj_singular @ maj_plural) + maj_unit_plural_nt = convert_space(maj_singular_nt @ maj_plural) + maj_unit_singular = convert_space(maj_singular) + maj_unit_singular_nt = convert_space(maj_singular_nt) + + self.maj_unit_plural = maj_unit_plural + self.maj_unit_plural_nt = maj_unit_plural_nt + self.maj_unit_singular = maj_unit_singular + self.maj_unit_singular_nt = maj_unit_singular_nt + + graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_unit_singular + pynutil.insert("\"") + graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_unit_plural + pynutil.insert("\"") + graph_maj_singular_nt = pynutil.insert("currency_maj: \"") + maj_unit_singular_nt + pynutil.insert("\"") + graph_maj_plural_nt = pynutil.insert("currency_maj: \"") + maj_unit_plural_nt + pynutil.insert("\"") + + optional_delete_fractional_zeros = pynini.closure( + pynutil.delete(",") + pynini.closure(pynutil.delete("0"), 1), 0, 1 + ) + + graph_integer_sg_en = pynutil.insert("integer_part: \"") + pynini.cross("1", "en") + pynutil.insert("\"") + graph_integer_sg_ett = pynutil.insert("integer_part: \"") + pynini.cross("1", "ett") + pynutil.insert("\"") + # only for decimals where third decimal after comma is non-zero or with quantity + decimal_delete_last_zeros = ( + pynini.closure(NEMO_DIGIT | pynutil.delete(" ")) + + pynini.accep(",") + + pynini.closure(NEMO_DIGIT, 2) + + (NEMO_DIGIT - "0") + + pynini.closure(pynutil.delete("0")) + ) + decimal_with_quantity = NEMO_SIGMA + SE_ALPHA + + decimal_part = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final + decimal_part_ett = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final_ett + graph_decimal = pynini.union( + graph_maj_plural + ensure_space + decimal_part, + graph_maj_plural_nt + ensure_space + decimal_part_ett, + decimal_part_ett + ensure_space + graph_maj_plural_nt, + decimal_part + ensure_space + graph_maj_plural, + ) + + graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + graph_integer_ett = pynutil.insert("integer_part: \"") + cardinal_graph_ett + pynutil.insert("\"") + + graph_integer_only = graph_maj_singular + ensure_space + graph_integer_sg_en + graph_integer_only |= graph_maj_singular_nt + ensure_space + graph_integer_sg_ett + graph_integer_only |= graph_maj_plural + ensure_space + graph_integer + graph_integer_only |= graph_maj_plural_nt + ensure_space + graph_integer_ett + graph_integer_only |= graph_integer_sg_en + ensure_space + graph_maj_singular + graph_integer_only |= graph_integer_sg_ett + ensure_space + graph_maj_singular_nt + graph_integer_only |= graph_integer + ensure_space + graph_maj_plural + graph_integer_only |= graph_integer_ett + ensure_space + graph_maj_plural_nt + + final_graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_decimal + + # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits + # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 + # not accepted: 002, 00, 0, + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert(" currency_min: \"") + min_singular + pynutil.insert("\"") + graph_min_plural = pynutil.insert(" currency_min: \"") + min_plural + pynutil.insert("\"") + + maj_singular_labels_all = [(x[0], "okta") for x in maj_singular_labels] + # format ** dollars ** cent + decimal_graph_with_minor = None + integer_graph_reordered = None + decimal_default_reordered = None + for curr_symbol, one_form in maj_singular_labels_all: + preserve_order = pynutil.insert(" preserve_order: true") + if one_form == "en": + integer_plus_maj = graph_integer + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_plural) + integer_plus_maj |= ( + graph_integer_sg_en + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_singular) + ) + else: + integer_plus_maj = ( + graph_integer_ett + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_plural_nt) + ) + integer_plus_maj |= ( + graph_integer_sg_ett + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_singular_nt) + ) + + integer_plus_maj_with_comma = pynini.compose( + NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | delete_space), integer_plus_maj + ) + integer_plus_maj = pynini.compose(pynini.closure(NEMO_DIGIT) - "0", integer_plus_maj) + integer_plus_maj |= integer_plus_maj_with_comma + + # all of the minor currency units are "en" + graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "en") + graph_fractional_one = pynutil.insert("fractional_part: \"") + graph_fractional_one + pynutil.insert("\"") + graph_fractional = ( + two_digits_fractional_part + @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") + @ cardinal.graph_hundreds_component_at_least_one_non_zero_digit_en + ) + graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") + + fractional_plus_min = graph_fractional + ensure_space + (pynutil.insert(curr_symbol) @ graph_min_plural) + fractional_plus_min |= ( + graph_fractional_one + ensure_space + (pynutil.insert(curr_symbol) @ graph_min_singular) + ) + + decimal_graph_with_minor_curr = integer_plus_maj + pynini.cross(",", " ") + fractional_plus_min + + if not deterministic: + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(",", " ") + + pynutil.insert("fractional_part: \"") + + two_digits_fractional_part @ cardinal.graph_hundreds_component_at_least_one_non_zero_digit_en + + pynutil.insert("\""), + weight=0.0001, + ) + default_fraction_graph = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final + + decimal_graph_with_minor_curr |= ( + pynini.closure(pynutil.delete("0"), 0, 1) + pynutil.delete(",") + fractional_plus_min + ) + decimal_graph_with_minor_curr = ( + pynutil.delete(curr_symbol) + decimal_graph_with_minor_curr + preserve_order + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr).optimize() + ) + + if not deterministic: + integer_graph_reordered_curr = ( + pynutil.delete(curr_symbol) + integer_plus_maj + preserve_order + ).optimize() + + integer_graph_reordered = ( + integer_graph_reordered_curr + if integer_graph_reordered is None + else pynini.union(integer_graph_reordered, integer_graph_reordered_curr).optimize() + ) + decimal_default_reordered_curr = ( + pynutil.delete(curr_symbol) + + default_fraction_graph + + ensure_space + + pynutil.insert(curr_symbol) @ graph_maj_plural + ) + + decimal_default_reordered = ( + decimal_default_reordered_curr + if decimal_default_reordered is None + else pynini.union(decimal_default_reordered, decimal_default_reordered_curr) + ).optimize() + + # weight for SH + final_graph |= pynutil.add_weight(decimal_graph_with_minor, -0.0001) + + if not deterministic: + final_graph |= integer_graph_reordered | decimal_default_reordered + # to handle "$2.00" cases + final_graph |= pynini.compose( + NEMO_SIGMA + pynutil.delete(",") + pynini.closure(pynutil.delete("0"), 1), integer_graph_reordered + ) + final_graph = self.add_tokens(final_graph.optimize()) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/ordinal.py b/nemo_text_processing/text_normalization/se/taggers/ordinal.py new file mode 100644 index 000000000..bbbcfe7c2 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/ordinal.py @@ -0,0 +1,192 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA +from nemo_text_processing.text_normalization.se.utils import get_abs_path +from pynini.lib import pynutil + +zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) +digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) +ord_digit = pynini.invert(pynini.string_file(get_abs_path("data/ordinal/digit.tsv"))) + + +def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by space) + and converts to a string of digits: + "1 000" -> "1000" + Args: + fst: Any pynini.FstLike object. Function composes fst onto string parser fst + + Returns: + fst: A pynini.FstLike object + """ + exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string + + cardinal_separator = pynini.union(NEMO_SPACE, ".") + cardinal_string = pynini.closure( + NEMO_DIGIT, 1 + ) # For string w/o punctuation (used for page numbers, thousand series) + + cardinal_string |= ( + up_to_three_digits + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator)) + + exactly_three_digits + ) + + return cardinal_string @ fst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinals, e.g. + "1000" -> cardinal { integer: "duhat" } + "2 000 000" -> cardinal { integer: "guoktemiljovnna" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + # Any single digit + graph_digit = digit + digits_no_one = (NEMO_DIGIT - "1") @ graph_digit + ord_digits_no_one = (NEMO_DIGIT - "1") @ ord_digit + + graph_zero = zero + + teen = pynutil.delete("1") + digit + pynutil.insert("nuppelogát") + teen |= pynini.cross("10", "logát") + ties = digits_no_one + pynini.cross("0", "logát") + ties |= digits_no_one + pynutil.insert("logi") + ord_digit + + graph_tens = teen + graph_ties = ties + + self.tens = graph_tens.optimize() + self.ties = graph_ties.optimize() + + two_digit_non_zero = pynini.union(graph_tens, graph_ties, (pynutil.delete("0") + ord_digit)) + graph_two_digit_non_zero = pynini.union(ord_digit, two_digit_non_zero) + + self.two_digit_non_zero = graph_two_digit_non_zero.optimize() + + # Three digit strings + hundreds = digits_no_one + pynutil.insert("čuođi") + hundreds |= pynini.cross("1", "čuođi") + hundreds_ord = digits_no_one + pynutil.insert("čuođát") + hundreds_ord |= pynini.cross("1", "čuođát") + + final_hundreds = hundreds + two_digit_non_zero + final_hundreds |= hundreds_ord + pynutil.delete("00") + + graph_hundreds = pynini.union(final_hundreds, graph_two_digit_non_zero) + + self.hundreds = graph_hundreds.optimize() + + # For all three digit strings with leading zeroes (graph appends '0's to manage place in string) + graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + (graph_tens | graph_ties)) + + graph_hundreds_component_at_least_one_non_zero_digit = graph_hundreds_component | ( + pynutil.delete("00") + ord_digit + ) + graph_hundreds_component_at_least_one_non_zero_digit_no_one = graph_hundreds_component | ( + pynutil.delete("00") + ord_digits_no_one + ) + self.graph_hundreds_component_at_least_one_non_zero_digit = ( + graph_hundreds_component_at_least_one_non_zero_digit + ) + self.graph_hundreds_component_at_least_one_non_zero_digit_no_one = ( + graph_hundreds_component_at_least_one_non_zero_digit_no_one.optimize() + ) + + # from here on, construct as though cardinals + duhat = pynutil.insert("duhát") + duhat_cross = pynini.cross("001", "duhát") + if not deterministic: + duhat_cross |= pynini.cross("001", "duhát ") + + graph_thousands_component_at_least_one_non_zero_digit = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit, + graph_hundreds_component_at_least_one_non_zero_digit_no_one + + duhat + + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + duhat_cross + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + ) + self.graph_thousands_component_at_least_one_non_zero_digit = ( + graph_thousands_component_at_least_one_non_zero_digit + ) + + graph_thousands_component_at_least_one_non_zero_digit_no_one = pynini.union( + pynutil.delete("000") + graph_hundreds_component_at_least_one_non_zero_digit_no_one, + graph_hundreds_component_at_least_one_non_zero_digit_no_one + + duhat + + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + duhat_cross + (graph_hundreds_component_at_least_one_non_zero_digit | pynutil.delete("000")), + ) + + graph = cardinal.graph_higher + ( + graph_thousands_component_at_least_one_non_zero_digit | pynutil.delete("000000") + ) + + higher_endings = pynini.string_map( + [ + ("duhát", "duháhat"), + ("duhat", "duháhat"), + ("iljárda", "iljárddat"), + ("iljon", "iljovnnat"), + ("iljun", "iljovnnat"), + ("iljovdna", "iljovnnat"), + ("illiuvdna", "iljovnnat"), + ("iljovnna", "iljovnnat"), + ("illiuvnna", "iljovnnat"), + ] + ) + + self.graph = ( + ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) + @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) + @ NEMO_DIGIT ** 24 + @ graph + @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), SE_ALPHA, SE_ALPHA, NEMO_SIGMA + ) + ) + self.graph |= graph_zero + self.graph @= pynini.cdrewrite(higher_endings, "", "[EOS]", NEMO_SIGMA) + + self.graph_bare_ordinals = filter_punctuation(self.graph).optimize() + self.graph = (self.graph_bare_ordinals + pynutil.delete(".")).optimize() + + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/telephone.py b/nemo_text_processing/text_normalization/se/taggers/telephone.py new file mode 100644 index 000000000..6b82d36cd --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/telephone.py @@ -0,0 +1,140 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_SPACE, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.se.graph_utils import ensure_space +from nemo_text_processing.text_normalization.se.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.se.utils import get_abs_path +from pynini.lib import pynutil + + +class TelephoneFst(GraphFst): + """ + tfn. 08-789 52 25 + Finite state transducer for classifying telephone numbers, e.g. + 123-123-5678 -> { number_part: "ett två tre ett två tre fyra sex sju åtta" }. + + Swedish numbers are written in the following formats: + 0X-XXX XXX XX + 0X-XXX XX XX + 0X-XX XX XX + 0XX-XXX XX XX + 0XX-XX XX XX + 0XX-XXX XX + 0XXX-XX XX XX + 0XXX-XXX XX + + See: + https://en.wikipedia.org/wiki/National_conventions_for_writing_telephone_numbers#Sweden + https://codegolf.stackexchange.com/questions/195787/format-a-swedish-phone-number + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + cardinal = CardinalFst(deterministic) + add_separator = pynutil.insert(", ") + zero_space = cardinal.zero_space + digit = cardinal.digit + two_digits = cardinal.two_digits_read + three_digits = cardinal.three_digits_read + two_or_three_digits = (two_digits | three_digits).optimize() + one_two_or_three_digits = (digit | two_or_three_digits).optimize() + zero_after_country_code = pynini.union(pynini.cross("(0)", "nolla "), zero_space) + bracketed = pynutil.delete("(") + one_two_or_three_digits + pynutil.delete(")") + + zero = pynini.cross("0", "nolla") + digit |= zero + + special_numbers = pynini.string_file(get_abs_path("data/telephone/special_numbers.tsv")) + + telephone_abbr = pynini.string_file(get_abs_path("data/telephone/telephone_abbr.tsv")) + if not deterministic: + telephone_abbr = pynini.string_file(get_abs_path("data/telephone/telephone_abbr_nd.tsv")) + telephone_prompt = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv")) + prompt = pynutil.insert("prompt: \"") + telephone_prompt + pynutil.insert("\"") + prompt |= pynutil.insert("prompt: \"") + telephone_abbr + pynutil.insert("\"") + prompt |= pynutil.insert("prompt: \"") + telephone_prompt + NEMO_SPACE + telephone_abbr + pynutil.insert("\"") + + country_code = pynini.closure(pynini.cross("+", "plus "), 0, 1) + one_two_or_three_digits + country_code = pynutil.insert("country_code: \"") + country_code + pynutil.insert("\"") + + opt_dash = pynini.closure(pynutil.delete("-"), 0, 1) + area_part = zero_after_country_code + one_two_or_three_digits + opt_dash + add_separator + area_part |= bracketed + add_separator + + base_number_part = pynini.union( + two_digits + NEMO_SPACE + two_digits + NEMO_SPACE + two_digits + NEMO_SPACE + two_digits, + three_digits + NEMO_SPACE + three_digits + NEMO_SPACE + three_digits, + three_digits + NEMO_SPACE + three_digits + NEMO_SPACE + two_digits, + three_digits + NEMO_SPACE + two_digits + NEMO_SPACE + two_digits, + three_digits + NEMO_SPACE + two_digits + insert_space + two_digits, + two_digits + NEMO_SPACE + two_digits + NEMO_SPACE + two_digits, + two_digits + NEMO_SPACE + two_digits + insert_space + two_digits, + three_digits + NEMO_SPACE + two_digits, + ) + number_part = area_part + delete_space + base_number_part + + self.number_graph = number_part + number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") + extension = pynutil.insert("extension: \"") + one_two_or_three_digits + pynutil.insert("\"") + extension = pynini.closure(insert_space + extension, 0, 1) + # FIXME: Swedish + extension_sv = pynini.union("ankn", "ankn.", "anknytning") + extension_no = pynini.union("utv", "utv.", "utvidelse", "linje", "ext.") + extension_fi = pynini.union("alanumero") + extensions = extension_sv | extension_no | extension_fi + ext_prompt = NEMO_SPACE + pynutil.delete(extensions) + ensure_space + passable = pynini.union(":", ": ", " ") + prompt_pass = pynutil.delete(passable) + insert_space + + special_numbers = pynutil.insert("number_part: \"") + special_numbers + pynutil.insert("\"") + prompt = prompt + prompt_pass + graph = pynini.union( + country_code + ensure_space + number_part, + country_code + ensure_space + number_part + ext_prompt + extension, + number_part + ext_prompt + extension, + prompt + number_part, + prompt + special_numbers, + prompt + country_code + number_part, + prompt + country_code + number_part + ext_prompt + extension, + prompt + number_part + ext_prompt + extension, + ) + self.tel_graph = graph.optimize() + + # ip + ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv")) + ip_graph = one_two_or_three_digits + (pynini.cross(".", " čuokkis ") + one_two_or_three_digits) ** 3 + graph |= ( + pynini.closure( + pynutil.insert("country_code: \"") + ip_prompts + pynutil.insert("\"") + delete_extra_space, 0, 1 + ) + + pynutil.insert("number_part: \"") + + ip_graph.optimize() + + pynutil.insert("\"") + ) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/time.py b/nemo_text_processing/text_normalization/se/taggers/time.py new file mode 100644 index 000000000..9a7ef910a --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/time.py @@ -0,0 +1,217 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + convert_space, + delete_extra_space, + insert_space, +) +from nemo_text_processing.text_normalization.se.graph_utils import ensure_space +from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels +from pynini.lib import pynutil + +QUARTERS = {15: "kvárta badjel", 30: "beal", 45: "kvárta váile"} + + +def get_all_to_or_from_numbers(): + output = {} + for num, word in QUARTERS.items(): + current_past = [] + current_to = [] + for i in range(1, 60): + if i == num: + continue + elif i < num: + current_to.append((str(i), str(num - i))) + else: + current_past.append((str(i), str(i - num))) + output[word] = {} + output[word]["past"] = current_past + output[word]["to"] = current_to + return output + + +def get_all_to_or_from_fst(cardinal: GraphFst): + numbers = get_all_to_or_from_numbers() + output = {} + for key in numbers: + for when in ["past", "to"]: + output[key] = {} + map = pynini.string_map(numbers[key][when]) + output[key][when] = pynini.project(map, "input") @ map @ cardinal.graph + return output + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 12:30 e.m. est -> time { hours: "tolv" minutes: "trettio" suffix: "eftermiddag" zone: "e s t" } + 2.30 e.m. -> time { hours: "två" minutes: "trettio" suffix: "eftermiddag" } + 02.30 e.m. -> time { hours: "två" minutes: "trettio" suffix: "eftermiddag" } + 2.00 e.m. -> time { hours: "två" suffix: "eftermiddag" } + kl. 2 e.m. -> time { hours: "två" suffix: "eftermiddag" } + 02:00 -> time { hours: "två" } + 2:00 -> time { hours: "två" } + 10:00:05 e.m. -> time { hours: "tio" minutes: "noll" seconds: "fem" suffix: "eftermiddag" } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + time_zone_graph = pynini.string_file(get_abs_path("data/time/time_zone.tsv")) + klockan = pynini.string_file(get_abs_path("data/time/prefix.tsv")) + + # only used for < 1000 thousand -> 0 weight + cardinal = cardinal.graph + + labels_hour = [str(x) for x in range(0, 24)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | ( + pynini.closure(pynutil.delete("0"), 0, 1) + NEMO_DIGIT + ) + + time_sep = pynutil.delete(pynini.union(":", ".")) + klockan_graph_piece = pynutil.insert("hours: \"") + klockan + + graph_hour = delete_leading_zero_to_double_digit @ pynini.union(*labels_hour) @ cardinal + + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal + klockan_hour_graph = klockan_graph_piece + ensure_space + graph_hour + pynutil.insert("\"") + + final_graph_hour = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynutil.delete("0") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + if not deterministic: + final_graph_minute |= ( + pynutil.insert("minutes: \"") + + (pynutil.delete("0") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_minute |= ( + pynutil.insert("minutes: \"") + pynini.cross("00", "nolla nolla") + pynutil.insert("\"") + ) + final_graph_second = ( + pynutil.insert("seconds: \"") + + (pynutil.delete("0") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + if not deterministic: + final_graph_second |= ( + pynutil.insert("seconds: \"") + + (pynini.cross("0", "nolla") + insert_space + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_second |= ( + pynutil.insert("seconds: \"") + pynini.cross("00", "nolla nolla") + pynutil.insert("\"") + ) + final_time_zone = pynutil.insert("zone: \"") + convert_space(time_zone_graph) + pynutil.insert("\"") + final_time_zone_optional = pynini.closure(NEMO_SPACE + final_time_zone, 0, 1,) + + # 2:30 pm, 02:30, 2:00 + graph_hm_kl = ( + klockan_hour_graph + + time_sep + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + final_time_zone_optional + ) + graph_hm_sfx = ( + final_graph_hour + + time_sep + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + ensure_space + ) + graph_hm = graph_hm_kl | graph_hm_sfx + + # 10:30:05 pm, + graph_hms_sfx = ( + final_graph_hour + + time_sep + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + time_sep + + (pynini.cross("00", " seconds: \"nolla\"") | insert_space + final_graph_second) + + ensure_space + + final_time_zone + ) + graph_hms_sfx |= ( + final_graph_hour + + pynutil.delete(".") + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + pynutil.delete(".") + + (pynini.cross("00", " seconds: \"nolla\"") | insert_space + final_graph_second) + + ensure_space + + final_time_zone + ) + graph_hms_kl = ( + klockan_hour_graph + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"nolla\"") | insert_space + final_graph_second) + + final_time_zone_optional + ) + graph_hms_kl |= ( + klockan_hour_graph + + pynutil.delete(".") + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + pynutil.delete(".") + + (pynini.cross("00", " seconds: \"nolla\"") | insert_space + final_graph_second) + + final_time_zone_optional + ) + graph_hms = graph_hms_kl | graph_hms_sfx + if not deterministic: + graph_hms |= ( + final_graph_hour + + pynutil.delete(".") + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + pynutil.delete(".") + + (pynini.cross("00", " seconds: \"nolla\"") | insert_space + final_graph_second) + ) + graph_hms |= ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"nolla\"") | insert_space + final_graph_minute) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"nolla\"") | insert_space + final_graph_second) + ) + self.graph_hms = graph_hms + self.graph_hm = graph_hm + # 2 pm est + ins_minutes = pynutil.insert(" minutes: \"nolla\"") + graph_h = final_graph_hour + ins_minutes + ensure_space + graph_h |= klockan_hour_graph + ins_minutes + self.graph_h = graph_h + + final_graph = (graph_hm | graph_h | graph_hms).optimize() @ pynini.cdrewrite( + delete_extra_space, "", "", NEMO_SIGMA + ) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..6dad08718 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py @@ -0,0 +1,188 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import time + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.se.taggers.abbreviation import AbbreviationFst +from nemo_text_processing.text_normalization.se.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.se.taggers.date import DateFst +from nemo_text_processing.text_normalization.se.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.se.taggers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.se.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.se.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.se.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.se.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.se.taggers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.se.taggers.time import TimeFst +from nemo_text_processing.text_normalization.se.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.se.taggers.word import WordFst +from pynini.lib import pynutil + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, f"se_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far" + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f'ClassifyFst.fst was restored from {far_file}.') + else: + logging.info(f"Creating ClassifyFst grammars.") + + start_time = time.time() + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + logging.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") + + start_time = time.time() + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + ordinal_graph = ordinal.fst + logging.debug(f"ordinal: {time.time() - start_time: .2f}s -- {ordinal_graph.num_states()} nodes") + + # start_time = time.time() + # decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + # decimal_graph = decimal.fst + # logging.debug(f"decimal: {time.time() - start_time: .2f}s -- {decimal_graph.num_states()} nodes") + + # start_time = time.time() + # fraction = FractionFst(deterministic=deterministic, ordinal=ordinal, cardinal=cardinal) + # fraction_graph = fraction.fst + # logging.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") + + # start_time = time.time() + # measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + # measure_graph = measure.fst + # logging.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") + + start_time = time.time() + date_graph = DateFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic).fst + logging.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes") + + start_time = time.time() + time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst + logging.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") + + start_time = time.time() + telephone_graph = TelephoneFst(deterministic=deterministic).fst + logging.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") + + start_time = time.time() + electonic_graph = ElectronicFst(deterministic=deterministic).fst + logging.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") + + # start_time = time.time() + # money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst + # logging.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") + + start_time = time.time() + whitelist_graph = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ).fst + logging.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") + + start_time = time.time() + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + logging.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") + + start_time = time.time() + word_graph = WordFst(deterministic=deterministic).fst + logging.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + # | pynutil.add_weight(decimal_graph, 1.1) + # | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + # | pynutil.add_weight(money_graph, 1.1) + # | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(electonic_graph, 1.1) + # | pynutil.add_weight(fraction_graph, 1.1) + ) + + if not deterministic: + abbreviation_graph = AbbreviationFst(whitelist, deterministic=deterministic).fst + classify |= pynutil.add_weight(abbreviation_graph, 100) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct), + 1, + ) + + classify |= pynutil.add_weight(word_graph, 100) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure( + ( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + | (pynutil.insert(" ") + punct + pynutil.insert(" ")) + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph |= punct + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/se/taggers/whitelist.py b/nemo_text_processing/text_normalization/se/taggers/whitelist.py new file mode 100644 index 000000000..fe74701d6 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/whitelist.py @@ -0,0 +1,64 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space +from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels +from pynini.lib import pynutil + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + "s:t" -> tokens { name: "sankt" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + # if not deterministic: + # units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measure/unit.tsv")) + # units_graph |= _get_whitelist_graph(input_case, file=get_abs_path("data/measure/unit_neuter.tsv")) + # units_graph |= _get_whitelist_graph(input_case, file=get_abs_path("data/abbreviations_nondet.tsv")) + # graph |= units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/word.py b/nemo_text_processing/text_normalization/se/taggers/word.py new file mode 100644 index 000000000..98e211c19 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/taggers/word.py @@ -0,0 +1,32 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst +from pynini.lib import pynutil + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. hund -> tokens { name: "hund" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/se/utils.py b/nemo_text_processing/text_normalization/se/utils.py new file mode 100644 index 000000000..91262904d --- /dev/null +++ b/nemo_text_processing/text_normalization/se/utils.py @@ -0,0 +1,72 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import logging +import os + +CASE_KEYS = ["ess", "com_pl", "com_sg", "gen_sg", "gen_pl", "ill_pl", "ill_sg", "loc_sg", "nom_pl"] +CASE_KEYS_EXT = CASE_KEYS + ["nom_sg"] + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path + + if not os.path.exists(abs_path): + logging.warning(f'{abs_path} does not exist') + return abs_path + + +def load_labels(abs_path): + """ + loads relative path file as list of lists + + Args: + abs_path: absolute path + + Returns list of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def load_case_forms(abs_path, extended=False): + """ + loads relative path file as dictionary, keyed on case/number + + Args: + abs_path: absolute path of file + + Returns dictionary of case forms + """ + KEYS = CASE_KEYS + if extended: + KEYS = CASE_KEYS_EXT + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + ret = {} + for label in labels: + if label and (label[0] in KEYS): + ret[label[0]] = label[1] + return ret diff --git a/nemo_text_processing/text_normalization/se/verbalizers/__init__.py b/nemo_text_processing/text_normalization/se/verbalizers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/se/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/se/verbalizers/cardinal.py new file mode 100644 index 000000000..bd0551a15 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/cardinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from pynini.lib import pynutil + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { integer: "guokte" } -> "guokte" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "minus "), 0, 1) + self.optional_sign = optional_sign + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + self.numbers = integer + graph = optional_sign + self.numbers + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/date.py b/nemo_text_processing/text_normalization/se/verbalizers/date.py new file mode 100644 index 000000000..cd20405be --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/date.py @@ -0,0 +1,82 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_extra_space, + delete_space, +) +from pynini.lib import pynutil + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { month: "february" day: "five" year: "twenty twelve" preserve_order: true } -> february fifth twenty twelve + date { month: "skábmamánnu" day: "gávccát" year: "duhátovccičuođivihttalogiguhtta" preserve_order: true } -> "skábmamánu gávccát beaivi duhátovccičuođivihttalogiguhtta" + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + month_nom_to_gen_map = pynini.cdrewrite(pynini.cross("mánu", "mánnu"), "", "[EOS]", NEMO_SIGMA) + + month = pynini.closure(NEMO_NOT_QUOTE, 1) + day = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + month = ( + pynutil.delete("month:") + + delete_space + + pynutil.delete("\"") + + (month @ month_nom_to_gen_map) + + pynutil.delete("\"") + ) + + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete("\"") + + pynutil.insert("beaivi ") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + delete_space + + pynutil.delete("\"") + ) + + graph_mdy = month + delete_extra_space + day + pynini.closure(delete_extra_space + year, 0, 1) + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = (graph_mdy | year) + delete_space + optional_preserve_order + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/decimals.py b/nemo_text_processing/text_normalization/se/verbalizers/decimals.py new file mode 100644 index 000000000..b07a51d57 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/decimals.py @@ -0,0 +1,73 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from pynini.lib import pynutil + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones + decimal { integer_part: "un" quantity: "billón" } -> un billón + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "minus ") + delete_space, 0, 1) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + self.integer = integer + conjunction = pynutil.insert(" komma ") + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + self.optional_quantity = optional_quantity + + graph = optional_sign + pynini.union( + (integer + quantity), (integer + delete_space + fractional + optional_quantity) + ) + + self.numbers_only_quantity = ( + optional_sign + + pynini.union((integer + quantity), (integer + delete_space + fractional + quantity)).optimize() + ) + + self.graph = (graph + delete_preserve_order).optimize() + + self.numbers = graph + graph += delete_preserve_order + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/electronic.py b/nemo_text_processing/text_normalization/se/verbalizers/electronic.py new file mode 100644 index 000000000..5ff4a97a9 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/electronic.py @@ -0,0 +1,74 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, + insert_space, +) +from nemo_text_processing.text_normalization.se.utils import get_abs_path +from pynini.lib import pynutil + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic + e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c smarfe-a hotmail čuokkis com" + -> "a b c smarfe-a h o t m a i l čuokkis c o m" + -> "a b c smarfe-a hotmail čuokkis c o m" + -> "a b c at h o t m a i l čuokkis com" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) + zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) + + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) + server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) + domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + self.symbols = graph_symbols + + graph_digit = digit_no_zero | zero + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + ( + NEMO_NOT_QUOTE - pynini.accep(" ") + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + user_name = pynutil.delete("username: \"") + add_space_after_char() + pynutil.delete("\"") + user_name @= verbalize_characters + + convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common + domain = convert_defaults + pynini.closure(insert_space + convert_defaults) + domain @= verbalize_characters + + domain = pynutil.delete("domain: \"") + domain + pynutil.delete("\"") + protocol = ( + pynutil.delete("protocol: \"") + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete("\"") + ) + self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | ( + user_name + pynini.accep(" ") + pynutil.insert("smarfe-a ") + domain + ) + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/fraction.py b/nemo_text_processing/text_normalization/se/verbalizers/fraction.py new file mode 100644 index 000000000..27357ecb1 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/fraction.py @@ -0,0 +1,73 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space +from pynini.lib import pynutil + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "tjugotre" numerator: "fyra" denominator: "femtedel" } } -> + tjugotre och fyra femtedelar + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + plurals = pynini.string_map([("kvart", "kvartar"), ("halv", "halva"), ("del", "delar")]) + + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\" ") + + denominators_sg = pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + denominators_pl = ( + pynutil.delete("denominator: \"") + + (pynini.closure(NEMO_NOT_QUOTE) @ pynini.cdrewrite(plurals, "", "[EOS]", NEMO_SIGMA)) + + pynutil.delete("\"") + ) + self.denominators = denominators_sg | denominators_pl + + either_one = pynini.union("en", "ett") + numerator_one = pynutil.delete("numerator: \"") + pynutil.delete(either_one) + pynutil.delete("\" ") + if not deterministic: + numerator_one |= pynutil.delete("numerator: \"") + either_one + pynutil.delete("\" ") + insert_space + numerator_rest = ( + pynutil.delete("numerator: \"") + + ( + (pynini.closure(NEMO_NOT_QUOTE) - either_one) + @ pynini.cdrewrite(pynini.cross("ett", "en"), "[BOS]", "[EOS]", NEMO_SIGMA) + ) + + pynutil.delete("\" ") + ) + + graph_sg = numerator_one + denominators_sg + graph_pl = numerator_rest + insert_space + denominators_pl + graph = graph_sg | graph_pl + + conjunction = pynutil.insert("och ") + if not deterministic and not lm: + conjunction = pynini.closure(conjunction, 0, 1) + + integer = pynini.closure(integer + insert_space + conjunction, 0, 1) + + graph = integer + graph + + self.graph = graph + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/measure.py b/nemo_text_processing/text_normalization/se/verbalizers/measure.py new file mode 100644 index 000000000..4ca928008 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/measure.py @@ -0,0 +1,95 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space +from pynini.lib import pynutil + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } -> minus twelve kilograms + measure { decimal { integer_part: "twelve" fractional_part: "five" } units: "kilograms" } -> twelve point five kilograms + tokens { measure { units: "covid" decimal { integer_part: "nineteen" fractional_part: "five" } } } -> covid nineteen point five + + Args: + decimal: DecimalFst + cardinal: CardinalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + optional_sign = cardinal.optional_sign + unit = ( + pynutil.delete("units: \"") + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) + + pynutil.delete("\"") + + delete_space + ) + + if not deterministic: + unit |= pynini.compose(unit, pynini.cross(pynini.union("inch", "inches"), "\"")) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph_fraction = ( + pynutil.delete("fraction {") + delete_space + fraction.graph + delete_space + pynutil.delete("}") + ) + + graph = (graph_cardinal | graph_decimal | graph_fraction) + delete_space + insert_space + unit + + # SH adds "preserve_order: true" by default + preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + graph |= unit + insert_space + (graph_cardinal | graph_decimal) + delete_space + pynini.closure(preserve_order) + # for only unit + graph |= ( + pynutil.delete("cardinal { integer: \"-\"") + + delete_space + + pynutil.delete("}") + + delete_space + + unit + + pynini.closure(preserve_order) + ) + math = ( + pynutil.delete("units: \"math\" ") + + delete_space + + graph_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + graph |= math + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/money.py b/nemo_text_processing/text_normalization/se/verbalizers/money.py new file mode 100644 index 000000000..6ffa83cd2 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/money.py @@ -0,0 +1,72 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_preserve_order, +) +from pynini.lib import pynutil + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "twelve" fractional_part: "o five" currency: "dollars" } -> twelve o five dollars + + Args: + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + keep_space = pynini.accep(" ") + maj = pynutil.delete("currency_maj: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + min = pynutil.delete("currency_min: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + fractional_part = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + integer_part = decimal.integer + + # *** currency_maj + graph_integer = integer_part + keep_space + maj + + # *** currency_maj + (***) | ((and) *** current_min) + fractional = fractional_part + delete_extra_space + min + + # if not deterministic: + # fractional |= pynutil.insert("och ") + fractional + # fractional |= pynutil.insert("komma ") + fractional + + graph_integer_with_minor = integer_part + keep_space + maj + keep_space + fractional + delete_preserve_order + + # *** point *** currency_maj + graph_decimal = decimal.numbers + keep_space + maj + + # *** current_min + graph_minor = fractional_part + delete_extra_space + min + delete_preserve_order + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + + if not deterministic: + graph |= graph_integer + delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/se/verbalizers/ordinal.py new file mode 100644 index 000000000..99e1592e5 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/ordinal.py @@ -0,0 +1,46 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from pynini.lib import pynutil + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "gávccát" } } -> gávccát + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + # TODO: add case forms? + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + # self.graph = pynini.compose(graph, suffix) + self.graph = graph + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/telephone.py b/nemo_text_processing/text_normalization/se/verbalizers/telephone.py new file mode 100644 index 000000000..7aa5bf334 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/telephone.py @@ -0,0 +1,73 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from pynini.lib import pynutil + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone numbers, e.g. + telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } + -> one, one two three, one two three, five six seven eight, one + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + country_code = pynutil.delete("country_code: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + optional_country_code = pynini.closure(country_code + delete_space + insert_space, 0, 1,) + + prompt_part = ( + pynutil.delete("prompt: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete("\"") + ) + NEMO_SPACE + + number_part = ( + pynutil.delete("number_part: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete("\"") + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynini.cross("extension: \"", "anknytning ") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\""), + 0, + 1, + ) + + graph = pynini.union( + prompt_part + optional_country_code + number_part + optional_extension, + optional_country_code + number_part + optional_extension, + ) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/time.py b/nemo_text_processing/text_normalization/se/verbalizers/time.py new file mode 100644 index 000000000..93e0434b1 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/time.py @@ -0,0 +1,89 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from pynini.lib import pynutil + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "tolv" minutes: "trettio" suffix: "förmiddag" zone: "e s t" } -> tolv trettio förmiddag e s t + time { hours: "tolv" } -> tolv + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + ANY_NOT_QUOTE = pynini.closure(NEMO_NOT_QUOTE, 1) + NOT_NOLL = pynini.difference(ANY_NOT_QUOTE, "nolla") + hour = pynutil.delete("hours:") + delete_space + pynutil.delete("\"") + ANY_NOT_QUOTE + pynutil.delete("\"") + minute = pynutil.delete("minutes:") + delete_space + pynutil.delete("\"") + NOT_NOLL + pynutil.delete("\"") + minute |= ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynutil.delete("nolla") + + pynutil.delete("\"") + ) + if not deterministic: + minute |= ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.cross("nolla", "nolla nolla") + + pynutil.delete("\"") + ) + suffix = pynutil.delete("suffix:") + delete_space + pynutil.delete("\"") + ANY_NOT_QUOTE + pynutil.delete("\"") + optional_suffix = pynini.closure(delete_space + insert_space + suffix, 0, 1) + zone = ( + pynutil.delete("zone:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1) + second = ( + pynutil.delete("seconds:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + graph = hour + NEMO_SPACE + minute + optional_suffix + optional_zone + graph |= hour + NEMO_SPACE + minute + NEMO_SPACE + second + optional_suffix + optional_zone + graph |= hour + NEMO_SPACE + suffix + optional_zone + graph |= hour + optional_zone + graph = ( + graph + @ pynini.cdrewrite(delete_extra_space, "", "", NEMO_SIGMA) + @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) + ) + # graph |= graph_hms + self.graph = graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/se/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/se/verbalizers/verbalize.py new file mode 100644 index 000000000..a2b12b2e7 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/verbalize.py @@ -0,0 +1,73 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.se.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.se.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.se.verbalizers.decimals import DecimalFst +from nemo_text_processing.text_normalization.se.verbalizers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.se.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.se.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.se.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.se.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.se.verbalizers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.se.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + fraction = FractionFst(deterministic=deterministic) + fraction_graph = fraction.fst + date = DateFst(deterministic=deterministic) + date_graph = date.fst + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst + telephone_graph = TelephoneFst(deterministic=deterministic).fst + time_graph = TimeFst(deterministic=deterministic).fst + + graph = ( + cardinal_graph + | measure_graph + | decimal_graph + | ordinal_graph + | date_graph + | electronic_graph + | money_graph + | fraction_graph + | whitelist_graph + | telephone_graph + | time_graph + ) + self.fst = graph diff --git a/nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py new file mode 100644 index 000000000..d0f4d82a6 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from nemo_text_processing.text_normalization.en.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.se.verbalizers.verbalize import VerbalizeFst +from pynini.lib import pynutil + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"sv_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/tests/nemo_text_processing/se/__init__.py b/tests/nemo_text_processing/se/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/tests/nemo_text_processing/se/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/se/data_text_normalization/__init__.py b/tests/nemo_text_processing/se/data_text_normalization/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..bfcc14c63 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,62 @@ +2~guokte +3~golbma +4~njeallje +5~vihtta +6~guhtta +7~čieža +8~gávcci +9~ovcci +10~logi +11~oktanuppelohkái +12~guoktenuppelohkái +13~golbmanuppelohkái +14~njealljenuppelohkái +15~vihttanuppelohkái +16~guhttanuppelohkái +17~čiežanuppelohkái +18~gávccinuppelohkái +19~ovccinuppelohkái +20~guoktelogi +22~guoktelogiguokte +23~guoktelogigolbma +24~guokteloginjeallje +25~guoktelogivihtta +26~guoktelogiguhtta +27~guoktelogičieža +28~guoktelogigávcci +29~guoktelogiovcci +30~golbmalogi +40~njealljelogi +50~vihttalogi +60~guhttalogi +70~čiežalogi +80~gávccilogi +90~ovccilogi +100~čuođi +102~čuođiguokte +120~čuođiguoktelogi +130~čuođigolbmalogi +1000~duhát +1 000~duhát +1.000~duhát +1010~duhátlogi +1020~duhátguoktelogi +1100~duhátčuođi +1110~duhátčuođilogi +1111~duhátčuođioktanuppelohkái +2000~guokteduhát +2002~guokteduhátguokte +2010~guokteduhátlogi +2020~guokteduhátguoktelogi +2100~guokteduhátčuođi +2110~guokteduhátčuođilogi +2111~guokteduhátčuođioktanuppelohkái +10000~logiduhát +10 000~logiduhát +10.000~logiduhát +100000~čuođiduhát +100 000~čuođiduhát +100.000~čuođiduhát +1 000 000~miljovdna +1.000.000~miljovdna +1 034 068~miljovdnagolbmaloginjealljeduhátguhttalogigávcci diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..693f77200 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_date.txt @@ -0,0 +1 @@ +cuoŋománu 22. b. 2023~cuoŋománnu guokteloginubbi beaivi guokteduhátguoktelogigolbma diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..f69ef4eb2 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,12 @@ +a.bc@gmail.com~a čuokkis b c smarfe-a g mail čuokkis com +cdf@abc.edu~c d f smarfe-a a b c čuokkis e d u +abc@gmail.abc~a b c smarfe-a g mail čuokkis a b c +abc@abc.com~a b c smarfe-a a b c čuokkis com +asdf123@abc.com~a s d f okta guokte golbma smarfe-a a b c čuokkis com +a1b2@abc.com~a okta b guokte smarfe-a a b c čuokkis com +ab3.sdd.3@gmail.com~a b golbma čuokkis s d d čuokkis golbma smarfe-a g mail čuokkis com +https://www.nvidia.com~h t t p s duppalčuokkis finjusáhcu finjusáhcu w w w čuokkis nvidia čuokkis com +www.nvidia.com~w w w čuokkis nvidia čuokkis com +www.abc.es/efg~w w w čuokkis a b c čuokkis e s finjusáhcu e f g +www.abc.es~w w w čuokkis a b c čuokkis e s +http://www.ourdailynews.com.sm~h t t p duppalčuokkis finjusáhcu finjusáhcu w w w čuokkis o u r d a i l y n e w s čuokkis com čuokkis s m diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..ec6d620c3 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,38 @@ +1.~vuosttaš +2.~nubbi +3.~goalmmát +4.~njealját +5.~viđat +6.~guđat +7.~čihččet +8.~gávccát +9.~ovccát +10.~logát +11.~oktanuppelogát +12.~guoktenuppelogát +13.~golbmanuppelogát +14.~njealljenuppelogát +15.~vihttanuppelogát +16.~guhttanuppelogát +17.~čiežanuppelogát +18.~gávccinuppelogát +19.~ovccinuppelogát +20.~guoktelogát +21.~guoktelogivuosttaš +22.~guokteloginubbi +23.~guoktelogigoalmmát +24.~guokteloginjealját +25.~guoktelogiviđat +26.~guoktelogiguđat +27.~guoktelogičihččet +28.~guoktelogigávccát +29.~guoktelogiovccát +30.~golbmalogát +40.~njealljelogát +50.~vihttalogát +60.~guhttalogát +70.~čiežalogát +80.~gávccilogát +90.~ovccilogát +100.~čuođát +1000.~duháhat diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..dcc3c61c4 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_time.txt @@ -0,0 +1,2 @@ +dii. 13:30~diibmu golbmanuppelohkái golbmalogi +13:30 cst~golbmanuppelohkái golbmalogi c s t diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..9dc1f9641 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1 @@ +tel.~telefonnummár diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..0d3e97450 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_word.txt @@ -0,0 +1,48 @@ +~ +yahoo!~yahoo! +veinte!~veinte! +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang diff --git a/tests/nemo_text_processing/se/test_cardinal.py b/tests/nemo_text_processing/se/test_cardinal.py new file mode 100644 index 000000000..11cd08d26 --- /dev/null +++ b/tests/nemo_text_processing/se/test_cardinal.py @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer(input_case='cased', lang='se', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/se/test_date.py b/tests/nemo_text_processing/se/test_date.py new file mode 100644 index 000000000..745a45d2c --- /dev/null +++ b/tests/nemo_text_processing/se/test_date.py @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer(input_case='cased', lang='se', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/se/test_electronic.py b/tests/nemo_text_processing/se/test_electronic.py new file mode 100644 index 000000000..1d0ea03e8 --- /dev/null +++ b/tests/nemo_text_processing/se/test_electronic.py @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestElectronic: + se_normalizer = Normalizer(input_case='cased', lang='se', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.se_normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/se/test_ordinal.py b/tests/nemo_text_processing/se/test_ordinal.py new file mode 100644 index 000000000..2008c8201 --- /dev/null +++ b/tests/nemo_text_processing/se/test_ordinal.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + normalizer = Normalizer(input_case='cased', lang='se', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/se/test_time.py b/tests/nemo_text_processing/se/test_time.py new file mode 100644 index 000000000..ae43cec51 --- /dev/null +++ b/tests/nemo_text_processing/se/test_time.py @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + se_normalizer = Normalizer(input_case='cased', lang='se', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.se_normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/se/test_whitelist.py b/tests/nemo_text_processing/se/test_whitelist.py new file mode 100644 index 000000000..bf208fd1b --- /dev/null +++ b/tests/nemo_text_processing/se/test_whitelist.py @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer(input_case='cased', lang='se', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/se/test_word.py b/tests/nemo_text_processing/se/test_word.py new file mode 100644 index 000000000..2193487e2 --- /dev/null +++ b/tests/nemo_text_processing/se/test_word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from nemo_text_processing.text_normalization.normalize import Normalizer +from parameterized import parameterized + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer(input_case='cased', lang='se', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected From bc84d2958b0b9068ab5420246e34c70cff578715 Mon Sep 17 00:00:00 2001 From: Katri Hiovain-Asikainen Date: Tue, 28 Mar 2023 11:11:36 +0000 Subject: [PATCH 2/3] Northern Sami symbol names Signed-off-by: Katri Hiovain-Asikainen Signed-off-by: Jim O'Regan --- .../se/data/electronic/symbols.tsv | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv b/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv index c86358fb5..33e616872 100644 --- a/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv +++ b/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv @@ -1,21 +1,22 @@ . čuokkis : duppalčuokkis - goallossáhcu -_ ! čuorvvasmearka ; beallečuokkis -# -$ -% -& ' apostrofa -* -+ / finjusáhcu -= ? gažaldatmearka -^ -{ -} -~ , rihkku +_ vuollesárggis +! čuorvvasmearka +# fáddágilkor +$ dollár +% proseanta +& ja +* asterisk ++ plussa += ovttamađodatmearka +^ sirkumfleaksa +{ gurut bárroruohtu +} olgeš bárroruohtu +~ tilde From e110bece5aeb2561edbbbb74b1c63c1f9252711e Mon Sep 17 00:00:00 2001 From: Jim O'Regan Date: Sun, 16 Aug 2026 19:27:45 +0200 Subject: [PATCH 3/3] Extend Northern Sami text normalization Signed-off-by: Jim O'Regan --- .pre-commit-config.yaml | 17 +- CHANGELOG.md | 51 + CONTRIBUTING.md | 33 +- Jenkinsfile | 409 +- README.md | 4 +- .../fst_alignment/alignment.py | 8 +- nemo_text_processing/hybrid/mlm_scorer.py | 2 +- nemo_text_processing/hybrid/model_utils.py | 17 +- nemo_text_processing/hybrid/utils.py | 30 +- .../hybrid/wfst_lm_rescoring.py | 5 +- .../inverse_text_normalization/ar/__init__.py | 6 +- .../ar/taggers/__init__.py | 6 +- .../ar/taggers/cardinal.py | 11 +- .../ar/taggers/decimal.py | 7 +- .../ar/taggers/fraction.py | 5 +- .../ar/taggers/measure.py | 3 +- .../ar/taggers/money.py | 3 +- .../ar/taggers/punctuation.py | 3 +- .../ar/taggers/tokenize_and_classify.py | 10 +- .../ar/taggers/word.py | 3 +- .../inverse_text_normalization/ar/utils.py | 4 +- .../ar/verbalizers/__init__.py | 4 +- .../ar/verbalizers/cardinal.py | 3 +- .../ar/verbalizers/decimal.py | 3 +- .../ar/verbalizers/fraction.py | 3 +- .../ar/verbalizers/measure.py | 3 +- .../ar/verbalizers/money.py | 3 +- .../ar/verbalizers/verbalize_final.py | 5 +- .../ar/verbalizers/word.py | 5 +- .../de/taggers/cardinal.py | 7 +- .../de/taggers/date.py | 3 +- .../de/taggers/decimal.py | 3 +- .../de/taggers/electronic.py | 5 +- .../de/taggers/fraction.py | 5 +- .../de/taggers/measure.py | 3 +- .../de/taggers/money.py | 3 +- .../de/taggers/ordinal.py | 3 +- .../de/taggers/telephone.py | 7 +- .../de/taggers/time.py | 5 +- .../de/taggers/tokenize_and_classify.py | 10 +- .../de/taggers/whitelist.py | 3 +- .../de/verbalizers/cardinal.py | 3 +- .../de/verbalizers/decimal.py | 3 +- .../de/verbalizers/measure.py | 3 +- .../de/verbalizers/money.py | 3 +- .../de/verbalizers/time.py | 5 +- .../de/verbalizers/verbalize_final.py | 5 +- .../en/clean_eval_data.py | 4 +- .../en/taggers/cardinal.py | 10 +- .../en/taggers/date.py | 11 +- .../en/taggers/decimal.py | 9 +- .../en/taggers/electronic.py | 11 +- .../en/taggers/measure.py | 7 +- .../en/taggers/money.py | 5 +- .../en/taggers/ordinal.py | 3 +- .../en/taggers/punctuation.py | 3 +- .../en/taggers/telephone.py | 19 +- .../en/taggers/time.py | 25 +- .../en/taggers/tokenize_and_classify.py | 10 +- .../en/taggers/whitelist.py | 3 +- .../en/taggers/word.py | 3 +- .../inverse_text_normalization/en/utils.py | 4 +- .../en/verbalizers/cardinal.py | 3 +- .../en/verbalizers/date.py | 3 +- .../en/verbalizers/decimal.py | 3 +- .../en/verbalizers/electronic.py | 3 +- .../en/verbalizers/fraction.py | 2 +- .../en/verbalizers/measure.py | 3 +- .../en/verbalizers/money.py | 3 +- .../en/verbalizers/ordinal.py | 3 +- .../en/verbalizers/telephone.py | 5 +- .../en/verbalizers/time.py | 3 +- .../en/verbalizers/verbalize_final.py | 5 +- .../en/verbalizers/whitelist.py | 5 +- .../en/verbalizers/word.py | 5 +- .../es/data/dates/months_cased.tsv | 12 + .../es/data/dates/year_suffix_cased.tsv | 11 + .../es/data/measures/measurements_plural.tsv | 12 +- .../data/measures/measurements_singular.tsv | 12 +- .../es/data/money/currency_major_plural.tsv | 1 + .../currency_major_plural_capitalized.tsv | 75 + .../es/data/money/currency_major_singular.tsv | 1 + .../currency_major_singular_capitalized.tsv | 76 + .../es/data/ordinals/digit_capitalized.tsv | 22 + .../es/data/ordinals/hundreds_capitalized.tsv | 18 + .../es/data/ordinals/teen_capitalized.tsv | 60 + .../es/data/ordinals/ties_capitalized.tsv | 15 + .../es/data/ordinals/twenties_capitalized.tsv | 50 + .../es/data/roman/digit.tsv | 18 +- .../es/data/roman/hundreds.tsv | 18 +- .../es/data/roman/thousands.tsv | 6 +- .../es/data/roman/ties.tsv | 18 +- .../es/data/time/time_suffix_cased.tsv | 15 + .../es/data/time/time_zone.tsv | 84 +- .../es/data/time/time_zone_cased.tsv | 42 + .../es/data/whitelist.tsv | 59 +- .../es/graph_utils.py | 3 +- .../es/taggers/cardinal.py | 98 +- .../es/taggers/date.py | 27 +- .../es/taggers/decimal.py | 56 +- .../es/taggers/electronic.py | 131 +- .../es/taggers/fraction.py | 22 +- .../es/taggers/measure.py | 20 +- .../es/taggers/money.py | 46 +- .../es/taggers/ordinal.py | 38 +- .../es/taggers/punctuation.py | 3 +- .../es/taggers/telephone.py | 54 +- .../es/taggers/time.py | 88 +- .../es/taggers/tokenize_and_classify.py | 38 +- .../es/taggers/whitelist.py | 57 +- .../es/taggers/word.py | 3 +- .../inverse_text_normalization/es/utils.py | 2 +- .../es/verbalizers/cardinal.py | 3 +- .../es/verbalizers/date.py | 15 +- .../es/verbalizers/decimal.py | 3 +- .../es/verbalizers/electronic.py | 3 +- .../es/verbalizers/fraction.py | 3 +- .../es/verbalizers/measure.py | 3 +- .../es/verbalizers/money.py | 3 +- .../es/verbalizers/ordinal.py | 3 +- .../es/verbalizers/telephone.py | 5 +- .../es/verbalizers/time.py | 9 +- .../es/verbalizers/verbalize_final.py | 5 +- .../es/verbalizers/whitelist.py | 5 +- .../es/verbalizers/word.py | 5 +- .../es_en/data/es_whitelist.tsv | 30 +- .../es_en/graph_utils.py | 3 +- .../es_en/taggers/tokenize_and_classify.py | 35 +- .../inverse_text_normalization/es_en/utils.py | 2 +- .../es_en/verbalizers/verbalize.py | 7 +- .../es_en/verbalizers/verbalize_final.py | 5 +- .../fr/graph_utils.py | 13 +- .../fr/taggers/cardinal.py | 22 +- .../fr/taggers/date.py | 7 +- .../fr/taggers/decimal.py | 9 +- .../fr/taggers/electronic.py | 3 +- .../fr/taggers/fraction.py | 5 +- .../fr/taggers/measure.py | 3 +- .../fr/taggers/money.py | 3 +- .../fr/taggers/ordinal.py | 5 +- .../fr/taggers/punctuation.py | 5 +- .../fr/taggers/telephone.py | 5 +- .../fr/taggers/time.py | 3 +- .../fr/taggers/tokenize_and_classify.py | 10 +- .../fr/taggers/whitelist.py | 3 +- .../fr/taggers/word.py | 3 +- .../inverse_text_normalization/fr/utils.py | 2 +- .../fr/verbalizers/cardinal.py | 3 +- .../fr/verbalizers/date.py | 3 +- .../fr/verbalizers/decimal.py | 9 +- .../fr/verbalizers/electronic.py | 3 +- .../fr/verbalizers/fraction.py | 3 +- .../fr/verbalizers/measure.py | 3 +- .../fr/verbalizers/money.py | 3 +- .../fr/verbalizers/ordinal.py | 7 +- .../fr/verbalizers/telephone.py | 5 +- .../fr/verbalizers/time.py | 5 +- .../fr/verbalizers/verbalize_final.py | 5 +- .../fr/verbalizers/whitelist.py | 5 +- .../fr/verbalizers/word.py | 5 +- .../inverse_text_normalization/he/__init__.py | 13 + .../he/data/__init__.py | 13 + .../he/data/decimals/__init__.py | 13 + .../he/data/decimals/minutes_exception.tsv | 2 + .../he/data/measurements.tsv | 45 + .../he/data/months.tsv | 13 + .../he/data/months_name2number.tsv | 12 + .../he/data/months_ordinal2number.tsv | 12 + .../he/data/numbers/__init__.py | 13 + .../he/data/numbers/decimal_fractions.tsv | 6 + .../he/data/numbers/digit.tsv | 20 + .../he/data/numbers/hundreds_exception.tsv | 2 + .../he/data/numbers/millions_exception.tsv | 1 + .../he/data/numbers/teen.tsv | 21 + .../he/data/numbers/thousands.tsv | 8 + .../he/data/numbers/thousands_exception.tsv | 2 + .../he/data/numbers/ties.tsv | 8 + .../he/data/numbers/viable_hours.tsv | 15 + .../he/data/numbers/zero.tsv | 1 + .../he/data/ordinals/__init__.py | 13 + .../he/data/ordinals/digit.tsv | 10 + .../he/data/prefix.tsv | 17 + .../he/data/spaced_measurements.tsv | 17 + .../he/data/time/__init__.py | 13 + .../he/data/time/day_suffix.tsv | 2 + .../he/data/time/evening_suffix.tsv | 2 + .../he/data/time/hour_to_evening.tsv | 7 + .../he/data/time/hour_to_night.tsv | 9 + .../he/data/time/hour_to_noon.tsv | 7 + .../he/data/time/midnight_to_hour.tsv | 1 + .../he/data/time/minute_to.tsv | 58 + .../he/data/time/minute_to_verbose.tsv | 6 + .../he/data/time/minute_verbose.tsv | 8 + .../he/data/time/night_suffix.tsv | 1 + .../he/data/time/noon_suffix.tsv | 3 + .../he/data/time/time_suffix.tsv | 8 + .../he/data/time/to_hour.tsv | 13 + .../he/data/whitelist.tsv | 20 + .../he/graph_utils.py | 119 + .../he/taggers/__init__.py | 13 + .../he/taggers/cardinal.py | 158 + .../he/taggers/date.py | 106 + .../he/taggers/decimal.py | 144 + .../he/taggers/measure.py | 114 + .../he/taggers/ordinal.py | 43 + .../he/taggers/punctuation.py | 35 + .../he/taggers/time.py | 202 + .../he/taggers/tokenize_and_classify.py | 104 + .../he/taggers/whitelist.py | 55 + .../he/taggers/word.py | 31 + .../inverse_text_normalization/he/utils.py | 182 + .../he/verbalizers/__init__.py | 13 + .../he/verbalizers/cardinal.py | 84 + .../he/verbalizers/date.py | 120 + .../he/verbalizers/decimal.py | 90 + .../he/verbalizers/measure.py | 107 + .../he/verbalizers/ordinal.py | 38 + .../he/verbalizers/time.py | 95 + .../he/verbalizers/verbalize.py | 54 + .../he/verbalizers/verbalize_final.py | 44 + .../he/verbalizers/whitelist.py | 50 + .../he/verbalizers/word.py | 34 + .../inverse_text_normalization/hi/__init__.py | 17 + .../hi/data/__init__.py | 13 + .../hi/data/address/__init__.py | 13 + .../hi/data/address/cities.tsv | 36 + .../hi/data/address/context_cues.tsv | 85 + .../hi/data/address/digit_passthrough.tsv | 10 + .../hi/data/address/ordinals.tsv | 167 + .../hi/data/address/special_characters.tsv | 2 + .../hi/data/address/states.tsv | 36 + .../hi/data/date/__init__.py | 13 + .../hi/data/date/century.tsv | 5 + .../hi/data/date/months.tsv | 14 + .../hi/data/measure/__init__.py | 13 + .../hi/data/measure/measurements.tsv | 295 + .../hi/data/money/__init__.py | 13 + .../hi/data/money/currency.tsv | 181 + .../hi/data/numbers/__init__.py | 13 + .../hi/data/numbers/digit.tsv | 13 + .../hi/data/numbers/hundred.tsv | 1 + .../hi/data/numbers/labels_exception.tsv | 4 + .../hi/data/numbers/paune.tsv | 231 + .../hi/data/numbers/teens_and_ties.tsv | 220 + .../hi/data/numbers/thousands.tsv | 8 + .../hi/data/numbers/zero.tsv | 1 + .../hi/data/ordinals/__init__.py | 13 + .../hi/data/ordinals/digit.tsv | 6 + .../hi/data/ordinals/digit_fem.tsv | 6 + .../hi/data/ordinals/hundred_digit.tsv | 12 + .../hi/data/ordinals/hundred_digit_fem.tsv | 12 + .../hi/data/ordinals/morph_features.tsv | 6 + .../hi/data/ordinals/suffix.tsv | 4 + .../hi/data/ordinals/suffix_fem.tsv | 2 + .../hi/data/ordinals/teens_and_ties.tsv | 1 + .../hi/data/ordinals/teens_and_ties_fem.tsv | 1 + .../hi/data/telephone/__init__.py | 13 + .../hi/data/telephone/context_cues.tsv | 15 + .../hi/data/telephone/country_codes.tsv | 1 + .../hi/data/telephone/eng_digit.tsv | 9 + .../hi/data/telephone/eng_zero.tsv | 1 + .../telephone/teens_and_ties_eng_to_hin.tsv | 90 + .../hi/data/time/__init__.py | 13 + .../hi/data/time/hour_for_paune.tsv | 15 + .../hi/data/whitelist/__init__.py | 13 + .../hi/data/whitelist/whitelist.tsv | 7 + .../hi/graph_utils.py | 204 + .../hi/taggers/__init__.py | 13 + .../hi/taggers/cardinal.py | 262 + .../hi/taggers/date.py | 113 + .../hi/taggers/decimal.py | 127 + .../hi/taggers/fraction.py | 164 + .../hi/taggers/measure.py | 254 + .../hi/taggers/money.py | 254 + .../hi/taggers/ordinal.py | 88 + .../hi/taggers/punctuation.py | 36 + .../hi/taggers/telephone.py | 160 + .../hi/taggers/time.py | 158 + .../hi/taggers/tokenize_and_classify.py | 123 + .../hi/taggers/whitelist.py | 57 + .../hi/taggers/word.py | 31 + .../inverse_text_normalization/hi/utils.py | 63 + .../hi/verbalizers/__init__.py | 13 + .../hi/verbalizers/cardinal.py | 49 + .../hi/verbalizers/date.py | 152 + .../hi/verbalizers/decimal.py | 59 + .../hi/verbalizers/fraction.py | 47 + .../hi/verbalizers/measure.py | 85 + .../hi/verbalizers/money.py | 47 + .../hi/verbalizers/ordinal.py | 46 + .../hi/verbalizers/telephone.py | 54 + .../hi/verbalizers/time.py | 108 + .../hi/verbalizers/verbalize.py | 67 + .../hi/verbalizers/verbalize_final.py | 45 + .../hi/verbalizers/whitelist.py | 44 + .../hi/verbalizers/word.py | 39 + .../hi_en/__init__.py | 17 + .../hi_en/graph_utils.py | 13 + .../hi_en/taggers/__init__.py | 13 + .../hi_en/taggers/tokenize_and_classify.py | 173 + .../inverse_text_normalization/hi_en/utils.py | 27 + .../hi_en/verbalizers/__init__.py | 13 + .../hi_en/verbalizers/verbalize.py | 102 + .../hi_en/verbalizers/verbalize_final.py | 44 + .../inverse_text_normalization/hy/__init__.py | 13 + .../hy/data/__init__.py | 13 + .../hy/data/currency.tsv | 53 + .../hy/data/measurement_dates.tsv | 4 + .../hy/data/measurements.tsv | 48 + .../hy/data/numbers/__init__.py | 13 + .../hy/data/numbers/digit.tsv | 11 + .../hy/data/numbers/digits_no_one.tsv | 10 + .../hy/data/numbers/ties.tsv | 10 + .../hy/data/ordinals/__init__.py | 13 + .../hy/data/ordinals/digit.tsv | 10 + .../hy/data/time/__init__.py | 13 + .../hy/data/time/hours.tsv | 29 + .../hy/data/time/minute_to.tsv | 59 + .../hy/data/time/minutes.tsv | 65 + .../hy/data/time/minutes_to.tsv | 59 + .../hy/data/time/to_hour.tsv | 25 + .../hy/data/whitelist.tsv | 7 + .../hy/taggers/__init__.py | 13 + .../hy/taggers/cardinal.py | 101 + .../hy/taggers/decimal.py | 117 + .../hy/taggers/fraction.py | 44 + .../hy/taggers/measure.py | 112 + .../hy/taggers/money.py | 64 + .../hy/taggers/ordinal.py | 55 + .../hy/taggers/punctuation.py | 36 + .../hy/taggers/time.py | 76 + .../hy/taggers/tokenize_and_classify.py | 114 + .../hy/taggers/whitelist.py | 36 + .../hy/taggers/word.py | 31 + .../inverse_text_normalization/hy/utils.py | 60 + .../hy/verbalizers/__init__.py | 13 + .../hy/verbalizers/cardinal.py | 42 + .../hy/verbalizers/decimal.py | 58 + .../hy/verbalizers/fraction.py | 43 + .../hy/verbalizers/measure.py | 63 + .../hy/verbalizers/money.py | 48 + .../hy/verbalizers/ordinal.py | 48 + .../hy/verbalizers/time.py | 53 + .../hy/verbalizers/verbalize.py | 51 + .../hy/verbalizers/verbalize_final.py | 45 + .../hy/verbalizers/whitelist.py | 38 + .../hy/verbalizers/word.py | 34 + .../inverse_normalize.py | 60 +- .../inverse_text_normalization/ja/__init__.py | 17 + .../ja/data/__init__.py | 17 + .../ja/data/char/__init__.py | 13 + .../ja/data/char/charset_extension.tsv | 6 + .../charset_national_standard_2013_8105.tsv | 8105 +++++++++++++++++ .../ja/data/char/fullwidth_to_halfwidth.tsv | 92 + .../ja/data/char/oov_tags.tsv | 1 + .../ja/data/char/punctuations_zh.tsv | 72 + .../ja/data/char/upper_to_lower.tsv | 9 + .../ja/data/date.tsv | 15 + .../ja/data/day.tsv | 31 + .../ja/data/months.tsv | 12 + .../ja/data/numbers/__init__.py | 17 + .../ja/data/numbers/digit.tsv | 9 + .../ja/data/numbers/teen.tsv | 26 + .../ja/data/numbers/ties.tsv | 30 + .../ja/data/numbers/zero.tsv | 6 + .../ja/data/time_hours.tsv | 25 + .../ja/data/time_minutes_seconds.tsv | 60 + .../ja/data/whitelist.tsv | 234 + .../ja/graph_utils.py | 292 + .../ja/taggers/__init__.py | 17 + .../ja/taggers/cardinal.py | 158 + .../ja/taggers/date.py | 111 + .../ja/taggers/decimal.py | 68 + .../ja/taggers/fraction.py | 105 + .../ja/taggers/ordinal.py | 40 + .../ja/taggers/preprocessor.py | 51 + .../ja/taggers/punctuation.py | 36 + .../ja/taggers/time.py | 68 + .../ja/taggers/tokenize_and_classify.py | 110 + .../ja/taggers/whitelist.py | 36 + .../ja/taggers/word.py | 29 + .../inverse_text_normalization/ja/utils.py | 35 + .../ja/verbalizers/__init__.py | 17 + .../ja/verbalizers/cardinal.py | 64 + .../ja/verbalizers/date.py | 64 + .../ja/verbalizers/decimal.py | 59 + .../ja/verbalizers/fraction.py | 65 + .../ja/verbalizers/ordinal.py | 35 + .../ja/verbalizers/post_processing.py | 132 + .../ja/verbalizers/postprocessor.py | 81 + .../ja/verbalizers/time.py | 69 + .../ja/verbalizers/verbalize.py | 58 + .../ja/verbalizers/verbalize_final.py | 59 + .../ja/verbalizers/whitelist.py | 32 + .../ja/verbalizers/word.py} | 13 +- .../inverse_text_normalization/ko/__init__.py | 17 + .../ko/data/__init__.py | 13 + .../ko/data/currency.tsv | 9 + .../ko/data/measure_units.tsv | 61 + .../ko/data/months.tsv | 12 + .../ko/data/numbers/__init__.py | 13 + .../ko/data/numbers/digit.tsv | 9 + .../ko/data/numbers/zero.tsv | 1 + .../ko/data/ordinals/__init__.py | 13 + .../ko/data/ordinals/counter_suffix.tsv | 17 + .../ko/data/ordinals/digit.tsv | 9 + .../ko/data/time/__init__.py | 13 + .../ko/data/time/ten_prefix.tsv | 4 + .../ko/data/time/time_hours.tsv | 23 + .../ko/data/whitelist.tsv | 35 + .../ko/graph_utils.py | 292 + .../ko/taggers/__init__.py | 13 + .../ko/taggers/cardinal.py | 118 + .../ko/taggers/date.py | 61 + .../ko/taggers/decimal.py | 74 + .../ko/taggers/fraction.py | 139 + .../ko/taggers/measure.py | 130 + .../ko/taggers/money.py | 48 + .../ko/taggers/ordinal.py | 126 + .../ko/taggers/telephone.py | 57 + .../ko/taggers/time.py | 101 + .../ko/taggers/tokenize_and_classify.py | 127 + .../ko/taggers/whitelist.py | 36 + .../ko/taggers/word.py | 31 + .../inverse_text_normalization/ko/utils.py | 20 + .../ko/verbalizers/__init__.py | 13 + .../ko/verbalizers/cardinal.py | 46 + .../ko/verbalizers/date.py | 50 + .../ko/verbalizers/decimal.py | 52 + .../ko/verbalizers/fraction.py | 76 + .../ko/verbalizers/measure.py | 88 + .../ko/verbalizers/money.py | 57 + .../ko/verbalizers/ordinal.py | 47 + .../ko/verbalizers/telephone.py | 33 + .../ko/verbalizers/time.py | 88 + .../ko/verbalizers/verbalize.py | 85 + .../ko/verbalizers/verbalize_final.py | 45 + .../ko/verbalizers/whitelist.py | 32 + .../ko/verbalizers/word.py | 39 + .../inverse_text_normalization/mr/__init__.py | 17 + .../mr/data/__init__.py | 13 + .../mr/data/date/__init__.py | 13 + .../mr/data/date/dates.tsv | 31 + .../mr/data/date/months.tsv | 12 + .../mr/data/date/prefixes.tsv | 2 + .../mr/data/numbers/__init__.py | 13 + .../mr/data/numbers/digits.tsv | 9 + .../mr/data/numbers/hundred.tsv | 1 + .../mr/data/numbers/tens.tsv | 90 + .../mr/data/numbers/thousands.tsv | 4 + .../mr/data/numbers/zero.tsv | 1 + .../mr/data/time/__init__.py | 13 + .../mr/data/time/hours.tsv | 12 + .../mr/data/time/hours_to.tsv | 12 + .../mr/data/time/minutes.tsv | 60 + .../mr/data/time/minutes_to.tsv | 59 + .../mr/graph_utils.py | 194 + .../mr/taggers/__init__.py | 13 + .../mr/taggers/cardinal.py | 113 + .../mr/taggers/date.py | 65 + .../mr/taggers/decimal.py | 100 + .../mr/taggers/punctuation.py | 36 + .../mr/taggers/time.py | 117 + .../mr/taggers/tokenize_and_classify.py | 97 + .../mr/taggers/word.py | 31 + .../inverse_text_normalization/mr/utils.py | 60 + .../mr/verbalizers/__init__.py | 13 + .../mr/verbalizers/cardinal.py | 53 + .../mr/verbalizers/date.py | 92 + .../mr/verbalizers/decimal.py | 62 + .../mr/verbalizers/time.py | 55 + .../mr/verbalizers/verbalize.py | 37 + .../mr/verbalizers/verbalize_final.py | 45 + .../mr/verbalizers/word.py | 39 + .../pt/taggers/cardinal.py | 18 +- .../pt/taggers/date.py | 3 +- .../pt/taggers/decimal.py | 3 +- .../pt/taggers/electronic.py | 3 +- .../pt/taggers/measure.py | 3 +- .../pt/taggers/money.py | 3 +- .../pt/taggers/ordinal.py | 3 +- .../pt/taggers/punctuation.py | 3 +- .../pt/taggers/telephone.py | 3 +- .../pt/taggers/time.py | 3 +- .../pt/taggers/tokenize_and_classify.py | 10 +- .../pt/taggers/whitelist.py | 3 +- .../pt/taggers/word.py | 3 +- .../pt/verbalizers/cardinal.py | 3 +- .../pt/verbalizers/date.py | 3 +- .../pt/verbalizers/decimal.py | 3 +- .../pt/verbalizers/electronic.py | 3 +- .../pt/verbalizers/measure.py | 3 +- .../pt/verbalizers/money.py | 3 +- .../pt/verbalizers/ordinal.py | 3 +- .../pt/verbalizers/telephone.py | 3 +- .../pt/verbalizers/time.py | 3 +- .../pt/verbalizers/verbalize_final.py | 3 +- .../pt/verbalizers/whitelist.py | 5 +- .../pt/verbalizers/word.py | 5 +- .../ru/taggers/cardinal.py | 5 +- .../ru/taggers/date.py | 3 +- .../ru/taggers/decimals.py | 3 +- .../ru/taggers/electronic.py | 3 +- .../ru/taggers/measure.py | 3 +- .../ru/taggers/money.py | 3 +- .../ru/taggers/ordinal.py | 3 +- .../ru/taggers/telephone.py | 3 +- .../ru/taggers/time.py | 3 +- .../ru/taggers/tokenize_and_classify.py | 10 +- .../ru/taggers/whitelist.py | 3 +- .../ru/verbalizers/cardinal.py | 3 +- .../ru/verbalizers/date.py | 3 +- .../ru/verbalizers/decimal.py | 3 +- .../ru/verbalizers/electronic.py | 3 +- .../ru/verbalizers/measure.py | 3 +- .../ru/verbalizers/money.py | 3 +- .../ru/verbalizers/ordinal.py | 3 +- .../ru/verbalizers/telephone.py | 3 +- .../ru/verbalizers/time.py | 3 +- .../ru/verbalizers/verbalize_final.py | 5 +- .../run_evaluate.py | 42 +- .../sv/taggers/cardinal.py | 5 +- .../sv/taggers/date.py | 6 +- .../sv/taggers/decimal.py | 13 +- .../sv/taggers/electronic.py | 5 +- .../sv/taggers/fraction.py | 5 +- .../sv/taggers/ordinal.py | 3 +- .../sv/taggers/telephone.py | 7 +- .../sv/taggers/time.py | 5 +- .../sv/taggers/tokenize_and_classify.py | 10 +- .../sv/taggers/whitelist.py | 3 +- .../inverse_text_normalization/sv/utils.py | 2 +- .../sv/verbalizers/cardinal.py | 3 +- .../sv/verbalizers/date.py | 3 +- .../sv/verbalizers/decimal.py | 3 +- .../sv/verbalizers/time.py | 3 +- .../sv/verbalizers/verbalize_final.py | 5 +- .../vi/data/currency.tsv | 2 +- .../vi/data/electronic/symbols.tsv | 1 + .../vi/data/electronic/url_symbols.tsv | 8 + .../vi/data/numbers/digit_special.tsv | 3 + .../vi/data/time/zero_prefix.tsv | 3 + .../vi/graph_utils.py | 14 +- .../vi/taggers/cardinal.py | 103 +- .../vi/taggers/date.py | 215 +- .../vi/taggers/decimal.py | 135 +- .../vi/taggers/electronic.py | 89 +- .../vi/taggers/fraction.py | 26 +- .../vi/taggers/measure.py | 32 +- .../vi/taggers/money.py | 9 +- .../vi/taggers/ordinal.py | 15 +- .../vi/taggers/punctuation.py | 3 +- .../vi/taggers/telephone.py | 108 +- .../vi/taggers/time.py | 49 +- .../vi/taggers/tokenize_and_classify.py | 29 +- .../vi/taggers/whitelist.py | 3 +- .../vi/taggers/word.py | 3 +- .../vi/verbalizers/cardinal.py | 3 +- .../vi/verbalizers/date.py | 7 +- .../vi/verbalizers/decimal.py | 10 +- .../vi/verbalizers/electronic.py | 5 +- .../vi/verbalizers/fraction.py | 3 +- .../vi/verbalizers/measure.py | 42 +- .../vi/verbalizers/money.py | 10 +- .../vi/verbalizers/ordinal.py | 3 +- .../vi/verbalizers/telephone.py | 31 +- .../vi/verbalizers/time.py | 14 +- .../vi/verbalizers/verbalize_final.py | 3 +- .../vi/verbalizers/whitelist.py | 8 +- .../vi/verbalizers/word.py | 8 +- .../zh/graph_utils.py | 18 +- .../zh/taggers/cardinal.py | 42 +- .../zh/taggers/date.py | 7 +- .../zh/taggers/decimal.py | 40 +- .../zh/taggers/fraction.py | 5 +- .../zh/taggers/money.py | 7 +- .../zh/taggers/ordinal.py | 3 +- .../zh/taggers/punctuation.py | 3 +- .../zh/taggers/time.py | 3 +- .../zh/taggers/tokenize_and_classify.py | 23 +- .../zh/taggers/whitelist.py | 3 +- .../zh/taggers/word.py | 5 +- .../inverse_text_normalization/zh/utils.py | 18 +- .../zh/verbalizers/cardinal.py | 5 +- .../zh/verbalizers/date.py | 3 +- .../zh/verbalizers/decimal.py | 20 +- .../zh/verbalizers/fraction.py | 3 +- .../zh/verbalizers/money.py | 3 +- .../zh/verbalizers/ordinal.py | 3 +- .../zh/verbalizers/time.py | 3 +- .../zh/verbalizers/verbalize.py | 2 +- .../zh/verbalizers/verbalize_final.py | 7 +- .../zh/verbalizers/whitelist.py | 5 +- .../zh/verbalizers/word.py | 5 +- nemo_text_processing/package_info.py | 6 +- .../text_normalization/ar/__init__.py | 5 +- .../text_normalization/ar/graph_utils.py | 14 +- .../text_normalization/ar/taggers/__init__.py | 4 +- .../text_normalization/ar/taggers/cardinal.py | 5 +- .../text_normalization/ar/taggers/decimal.py | 32 +- .../text_normalization/ar/taggers/fraction.py | 5 +- .../text_normalization/ar/taggers/measure.py | 24 +- .../text_normalization/ar/taggers/money.py | 31 +- .../ar/taggers/tokenize_and_classify.py | 10 +- .../text_normalization/ar/taggers/word.py | 3 +- .../text_normalization/ar/utils.py | 8 +- .../ar/verbalizers/__init__.py | 4 +- .../ar/verbalizers/cardinal.py | 3 +- .../ar/verbalizers/decimal.py | 3 +- .../ar/verbalizers/fraction.py | 3 +- .../ar/verbalizers/measure.py | 5 +- .../ar/verbalizers/money.py | 16 +- .../ar/verbalizers/verbalize_final.py | 10 +- .../text_normalization/ar/verbalizers/word.py | 5 +- .../text_normalization/data_loader_utils.py | 49 +- .../de/data/electronic/domain.tsv | 843 +- .../de/data/electronic/server_name.tsv | 7 +- .../text_normalization/de/data/whitelist.tsv | 14 + .../text_normalization/de/taggers/cardinal.py | 9 +- .../text_normalization/de/taggers/date.py | 7 +- .../text_normalization/de/taggers/decimal.py | 7 +- .../de/taggers/electronic.py | 54 +- .../text_normalization/de/taggers/fraction.py | 3 +- .../text_normalization/de/taggers/measure.py | 9 +- .../text_normalization/de/taggers/money.py | 3 +- .../text_normalization/de/taggers/ordinal.py | 5 +- .../de/taggers/telephone.py | 9 +- .../text_normalization/de/taggers/time.py | 29 +- .../de/taggers/tokenize_and_classify.py | 28 +- .../de/taggers/whitelist.py | 3 +- .../text_normalization/de/taggers/word.py | 3 +- .../text_normalization/de/utils.py | 11 +- .../de/verbalizers/cardinal.py | 3 +- .../text_normalization/de/verbalizers/date.py | 3 +- .../de/verbalizers/decimal.py | 7 +- .../de/verbalizers/electronic.py | 24 +- .../de/verbalizers/fraction.py | 3 +- .../de/verbalizers/measure.py | 5 +- .../de/verbalizers/money.py | 3 +- .../de/verbalizers/ordinal.py | 8 +- .../de/verbalizers/telephone.py | 5 +- .../text_normalization/de/verbalizers/time.py | 3 +- .../de/verbalizers/verbalize_final.py | 10 +- .../text_normalization/en/clean_eval_data.py | 4 +- .../en/data/electronic/cc_cues.tsv | 5 + .../en/data/electronic/domain.tsv | 1 + .../en/data/measure/unit.tsv | 10 +- .../en/data/money/per_unit.tsv | 70 +- .../en/data/whitelist/tts.tsv | 93 +- .../text_normalization/en/graph_utils.py | 89 +- .../en/taggers/abbreviation.py | 3 +- .../text_normalization/en/taggers/cardinal.py | 11 +- .../text_normalization/en/taggers/date.py | 19 +- .../text_normalization/en/taggers/decimal.py | 7 +- .../en/taggers/electronic.py | 95 +- .../text_normalization/en/taggers/fraction.py | 3 +- .../text_normalization/en/taggers/measure.py | 103 +- .../text_normalization/en/taggers/money.py | 39 +- .../text_normalization/en/taggers/ordinal.py | 5 +- .../en/taggers/punctuation.py | 7 +- .../text_normalization/en/taggers/range.py | 18 +- .../text_normalization/en/taggers/roman.py | 3 +- .../text_normalization/en/taggers/serial.py | 164 +- .../en/taggers/telephone.py | 9 +- .../text_normalization/en/taggers/time.py | 5 +- .../en/taggers/tokenize_and_classify.py | 81 +- .../en/taggers/tokenize_and_classify_lm.py | 14 +- .../tokenize_and_classify_with_audio.py | 12 +- .../en/taggers/whitelist.py | 3 +- .../text_normalization/en/taggers/word.py | 5 +- .../text_normalization/en/utils.py | 8 +- .../en/verbalizers/abbreviation.py | 3 +- .../en/verbalizers/cardinal.py | 3 +- .../text_normalization/en/verbalizers/date.py | 5 +- .../en/verbalizers/decimal.py | 3 +- .../en/verbalizers/electronic.py | 5 +- .../en/verbalizers/fraction.py | 5 +- .../en/verbalizers/measure.py | 5 +- .../en/verbalizers/money.py | 22 +- .../en/verbalizers/ordinal.py | 8 +- .../en/verbalizers/post_processing.py | 15 +- .../en/verbalizers/roman.py | 3 +- .../en/verbalizers/telephone.py | 3 +- .../text_normalization/en/verbalizers/time.py | 11 +- .../en/verbalizers/verbalize_final.py | 8 +- .../en/verbalizers/whitelist.py | 5 +- .../text_normalization/en/verbalizers/word.py | 5 +- .../es/data/ordinals/roman_exceptions.tsv | 10 + .../text_normalization/es/data/whitelist.tsv | 2 - .../text_normalization/es/graph_utils.py | 92 +- .../text_normalization/es/taggers/cardinal.py | 7 +- .../text_normalization/es/taggers/date.py | 5 +- .../text_normalization/es/taggers/decimals.py | 3 +- .../es/taggers/electronic.py | 87 +- .../text_normalization/es/taggers/fraction.py | 59 +- .../text_normalization/es/taggers/measure.py | 7 +- .../text_normalization/es/taggers/money.py | 3 +- .../text_normalization/es/taggers/ordinal.py | 8 +- .../es/taggers/telephone.py | 7 +- .../text_normalization/es/taggers/time.py | 7 +- .../es/taggers/tokenize_and_classify.py | 34 +- .../es/taggers/whitelist.py | 3 +- .../text_normalization/es/taggers/word.py | 3 +- .../text_normalization/es/utils.py | 4 +- .../es/verbalizers/cardinal.py | 15 +- .../text_normalization/es/verbalizers/date.py | 3 +- .../es/verbalizers/decimals.py | 15 +- .../es/verbalizers/electronic.py | 41 +- .../es/verbalizers/fraction.py | 23 +- .../es/verbalizers/measure.py | 3 +- .../es/verbalizers/money.py | 3 +- .../es/verbalizers/ordinal.py | 5 +- .../es/verbalizers/telephone.py | 3 +- .../text_normalization/es/verbalizers/time.py | 3 +- .../es/verbalizers/verbalize_final.py | 8 +- .../fr/data/dates/__init__.py | 13 + .../text_normalization/fr/data/dates/eras.tsv | 8 + .../fr/data/dates/months.tsv | 12 + .../data/measures/__init__.py} | 0 .../fr/data/measures/measurements.tsv | 17 + .../text_normalization/fr/data/whitelist.tsv | 14 +- .../text_normalization/fr/taggers/cardinal.py | 3 +- .../text_normalization/fr/taggers/date.py | 97 + .../text_normalization/fr/taggers/decimals.py | 3 +- .../text_normalization/fr/taggers/fraction.py | 3 +- .../text_normalization/fr/taggers/ordinal.py | 5 +- .../fr/taggers/tokenize_and_classify.py | 34 +- .../fr/taggers/whitelist.py | 3 +- .../text_normalization/fr/taggers/word.py | 3 +- .../text_normalization/fr/utils.py | 4 +- .../fr/verbalizers/__init__.py | 26 +- .../fr/verbalizers/cardinal.py | 15 +- .../text_normalization/fr/verbalizers/date.py | 51 + .../fr/verbalizers/decimals.py | 15 +- .../fr/verbalizers/fraction.py | 17 +- .../fr/verbalizers/ordinal.py | 3 +- .../fr/verbalizers/verbalize.py | 5 +- .../fr/verbalizers/verbalize_final.py | 9 +- .../text_normalization/hi/__init__.py | 17 + .../text_normalization/hi/data/__init__.py | 13 + .../hi/data/address/__init__.py | 13 + .../hi/data/address/cities.tsv | 36 + .../hi/data/address/context.tsv | 47 + .../hi/data/address/letters.tsv | 26 + .../hi/data/address/special_characters.tsv | 2 + .../hi/data/address/states.tsv | 36 + .../hi/data/date/__init__.py | 13 + .../text_normalization/hi/data/date/days.tsv | 62 + .../hi/data/date/months.tsv | 24 + .../hi/data/date/prefixes.tsv | 4 + .../hi/data/date/suffixes.tsv | 10 + .../hi/data/date/year_suffix.tsv | 2 + .../hi/data/electronic/__init__.py | 13 + .../hi/data/electronic/domain.tsv | 24 + .../hi/data/electronic/elements.tsv | 132 + .../hi/data/electronic/file_extensions.tsv | 34 + .../hi/data/electronic/protocols.tsv | 5 + .../hi/data/electronic/subscript_digit.tsv | 10 + .../hi/data/electronic/symbol_classes.tsv | 16 + .../hi/data/electronic/symbols.tsv | 36 + .../hi/data/fraction/__init__.py | 13 + .../hi/data/fraction/common_fractions.tsv | 10 + .../hi/data/measure/__init__.py | 13 + .../hi/data/measure/quarterly_units_list.tsv | 5 + .../hi/data/measure/quarterly_units_map.tsv | 10 + .../hi/data/measure/unit.tsv | 145 + .../hi/data/measure/unit_year_formal.tsv | 1 + .../hi/data/money/__init__.py | 13 + .../hi/data/money/currency.tsv | 9 + .../hi/data/money/currency_singular.tsv | 9 + .../hi/data/money/major_minor_currencies.tsv | 10 + .../hi/data/numbers/__init__.py | 13 + .../hi/data/numbers/digit.tsv | 18 + .../hi/data/numbers/hundred.tsv | 1 + .../hi/data/numbers/teens_and_ties.tsv | 90 + .../hi/data/numbers/teens_and_ties_en.tsv | 90 + .../hi/data/numbers/thousands.tsv | 8 + .../hi/data/numbers/zero.tsv | 2 + .../hi/data/ordinal/__init__.py | 13 + .../hi/data/ordinal/en_to_hi_digit.tsv | 10 + .../hi/data/ordinal/exceptions.tsv | 50 + .../hi/data/ordinal/suffixes.tsv | 3 + .../hi/data/ordinal/suffixes_map.tsv | 1 + .../hi/data/roman/__init__.py | 13 + .../data/roman/roman_ordinal_exceptions.tsv | 10 + .../hi/data/roman/roman_to_spoken.tsv | 100 + .../hi/data/serial/__init__.py | 13 + .../hi/data/serial/chars.tsv | 68 + .../hi/data/serial/power_special.tsv | 4 + .../hi/data/serial/special_symbols.tsv | 4 + .../hi/data/telephone/__init__.py | 13 + .../hi/data/telephone/credit_context.tsv | 3 + .../hi/data/telephone/landline_context.tsv | 5 + .../hi/data/telephone/mobile_context.tsv | 4 + .../hi/data/telephone/number.tsv | 10 + .../hi/data/telephone/pincode_context.tsv | 4 + .../hi/data/time/__init__.py | 13 + .../text_normalization/hi/data/time/hours.tsv | 50 + .../hi/data/time/minutes.tsv | 120 + .../hi/data/time/seconds.tsv | 120 + .../hi/data/whitelist/__init__.py | 13 + .../hi/data/whitelist/abbreviations.tsv | 7 + .../hi/data/whitelist/paune_mappings.tsv | 100 + .../text_normalization/hi/graph_utils.py | 225 + .../text_normalization/hi/taggers/__init__.py | 13 + .../text_normalization/hi/taggers/cardinal.py | 395 + .../text_normalization/hi/taggers/date.py | 249 + .../text_normalization/hi/taggers/decimal.py | 84 + .../hi/taggers/electronic.py | 181 + .../text_normalization/hi/taggers/fraction.py | 140 + .../text_normalization/hi/taggers/measure.py | 477 + .../text_normalization/hi/taggers/money.py | 158 + .../text_normalization/hi/taggers/ordinal.py | 61 + .../hi/taggers/punctuation.py | 62 + .../text_normalization/hi/taggers/roman.py | 138 + .../text_normalization/hi/taggers/serial.py | 162 + .../hi/taggers/telephone.py | 228 + .../text_normalization/hi/taggers/time.py | 138 + .../hi/taggers/tokenize_and_classify.py | 181 + .../hi/taggers/whitelist.py | 73 + .../text_normalization/hi/taggers/word.py | 61 + .../text_normalization/hi/utils.py | 74 + .../hi/verbalizers/__init__.py | 13 + .../hi/verbalizers/cardinal.py | 49 + .../text_normalization/hi/verbalizers/date.py | 74 + .../hi/verbalizers/decimal.py | 53 + .../hi/verbalizers/electronic.py | 119 + .../hi/verbalizers/fraction.py | 58 + .../hi/verbalizers/measure.py | 86 + .../hi/verbalizers/money.py | 102 + .../hi/verbalizers/ordinal.py | 38 + .../hi/verbalizers/post_processing.py | 96 + .../hi/verbalizers/roman.py | 98 + .../hi/verbalizers/telephone.py | 72 + .../text_normalization/hi/verbalizers/time.py | 81 + .../hi/verbalizers/verbalize.py | 94 + .../hi/verbalizers/verbalize_final.py | 76 + .../hi/verbalizers/whitelist.py | 42 + .../text_normalization/hi/verbalizers/word.py | 43 + .../text_normalization/hu/graph_utils.py | 3 +- .../text_normalization/hu/taggers/cardinal.py | 13 +- .../text_normalization/hu/taggers/date.py | 9 +- .../text_normalization/hu/taggers/decimal.py | 9 +- .../hu/taggers/electronic.py | 96 +- .../text_normalization/hu/taggers/fraction.py | 3 +- .../text_normalization/hu/taggers/measure.py | 7 +- .../text_normalization/hu/taggers/money.py | 3 +- .../text_normalization/hu/taggers/ordinal.py | 5 +- .../hu/taggers/telephone.py | 7 +- .../text_normalization/hu/taggers/time.py | 11 +- .../hu/taggers/tokenize_and_classify.py | 33 +- .../hu/taggers/whitelist.py | 3 +- .../text_normalization/hu/taggers/word.py | 3 +- .../text_normalization/hu/utils.py | 6 +- .../hu/verbalizers/cardinal.py | 3 +- .../text_normalization/hu/verbalizers/date.py | 3 +- .../hu/verbalizers/decimal.py | 3 +- .../hu/verbalizers/electronic.py | 57 +- .../hu/verbalizers/fraction.py | 3 +- .../hu/verbalizers/measure.py | 5 +- .../hu/verbalizers/money.py | 3 +- .../hu/verbalizers/ordinal.py | 3 +- .../hu/verbalizers/telephone.py | 13 +- .../text_normalization/hu/verbalizers/time.py | 3 +- .../hu/verbalizers/verbalize_final.py | 9 +- .../text_normalization/hy/__init__.py | 13 + .../text_normalization/hy/data/__init__.py | 13 + .../text_normalization/hy/data/currency.tsv | 52 + .../hy/data/measurement_dates.tsv | 8 + .../hy/data/measurements.tsv | 44 + .../hy/data/numbers/__init__.py | 13 + .../hy/data/numbers/digit.tsv | 9 + .../hy/data/numbers/quantities.tsv | 3 + .../hy/data/numbers/ties.tsv | 8 + .../hy/data/ordinal/__init__.py | 13 + .../hy/data/ordinal/digit.tsv | 5 + .../hy/data/time/__init__.py | 13 + .../text_normalization/hy/data/time/hours.tsv | 26 + .../hy/data/time/minutes.tsv | 61 + .../text_normalization/hy/data/whitelist.tsv | 14 + .../text_normalization/hy/taggers/__init__.py | 13 + .../text_normalization/hy/taggers/cardinal.py | 88 + .../text_normalization/hy/taggers/decimal.py | 87 + .../text_normalization/hy/taggers/fraction.py | 37 + .../text_normalization/hy/taggers/measure.py | 99 + .../text_normalization/hy/taggers/money.py | 83 + .../text_normalization/hy/taggers/ordinal.py | 74 + .../hy/taggers/punctuation.py | 36 + .../text_normalization/hy/taggers/time.py | 38 + .../hy/taggers/tokenize_and_classify.py | 116 + .../hy/taggers/whitelist.py | 48 + .../text_normalization/hy/taggers/word.py | 30 + .../text_normalization/hy/utils.py | 43 + .../hy/verbalizers/__init__.py | 13 + .../hy/verbalizers/cardinal.py | 34 + .../hy/verbalizers/decimal.py | 63 + .../hy/verbalizers/fraction.py | 44 + .../hy/verbalizers/measure.py | 76 + .../hy/verbalizers/money.py | 39 + .../hy/verbalizers/ordinal.py | 36 + .../text_normalization/hy/verbalizers/time.py | 42 + .../hy/verbalizers/verbalize.py | 60 + .../hy/verbalizers/verbalize_final.py | 51 + .../hy/verbalizers/whitelist.py | 34 + .../text_normalization/hy/verbalizers/word.py | 38 + .../text_normalization/it/__init__.py | 4 + .../{electronic/__init__ .py => __init__.py} | 0 .../__init__ .py => electronic/__init__.py} | 0 .../it/data/electronic/domain.tsv | 3 +- .../it/data/measure/measurements.tsv | 3 +- .../__init__ .py => data/numbers/__init__.py} | 0 .../it/data/numbers/tens_eight.tsv | 8 + .../whitelist/__init__.py} | 0 .../it/data/{ => whitelist}/whitelist.tsv | 0 .../text_normalization/it/taggers/__init__.py | 13 + .../text_normalization/it/taggers/cardinal.py | 11 +- .../text_normalization/it/taggers/decimals.py | 6 +- .../it/taggers/electronic.py | 89 +- .../text_normalization/it/taggers/measure.py | 11 +- .../text_normalization/it/taggers/money.py | 5 +- .../text_normalization/it/taggers/time.py | 80 + .../it/taggers/tokenize_and_classify.py | 59 +- .../it/taggers/whitelist.py | 9 +- .../text_normalization/it/taggers/word.py | 3 +- .../text_normalization/it/utils.py | 6 +- .../it/verbalizers/__init__.py | 13 + .../it/verbalizers/cardinal.py | 3 +- .../it/verbalizers/decimal.py | 15 +- .../it/verbalizers/electronic.py | 41 +- .../it/verbalizers/measure.py | 5 +- .../it/verbalizers/money.py | 5 +- .../text_normalization/it/verbalizers/time.py | 85 + .../it/verbalizers/verbalize.py | 15 +- .../it/verbalizers/verbalize_final.py | 8 +- .../text_normalization/ja/__init__.py | 18 + .../text_normalization/ja/data/__init__.py | 13 + .../ja/data/date/__init__.py | 13 + .../text_normalization/ja/data/date/day.tsv | 31 + .../text_normalization/ja/data/date/era.tsv | 12 + .../ja/data/date/era_abbrev.tsv | 5 + .../text_normalization/ja/data/date/month.tsv | 12 + .../text_normalization/ja/data/date/week.tsv | 15 + .../ja/data/numbers/__init__.py | 13 + .../ja/data/numbers/digit.tsv | 9 + .../ja/data/numbers/teen.tsv | 10 + .../ja/data/numbers/ties.tsv | 8 + .../ja/data/numbers/zero.tsv | 1 + .../text_normalization/ja/data/symbol.tsv | 23 + .../ja/data/time/__init__.py | 13 + .../ja/data/time/division.tsv | 23 + .../text_normalization/ja/data/time/hour.tsv | 24 + .../ja/data/time/minute.tsv | 60 + .../ja/data/time/second.tsv | 60 + .../text_normalization/ja/data/whitelist.tsv | 40 + .../text_normalization/ja/graph_utils.py | 297 + .../text_normalization/ja/taggers/__init__.py | 13 + .../text_normalization/ja/taggers/cardinal.py | 191 + .../text_normalization/ja/taggers/date.py | 227 + .../text_normalization/ja/taggers/decimal.py | 59 + .../text_normalization/ja/taggers/fraction.py | 125 + .../text_normalization/ja/taggers/ordinal.py | 42 + .../ja/taggers/punctuation.py | 70 + .../text_normalization/ja/taggers/time.py | 112 + .../ja/taggers/tokenize_and_classify.py | 94 + .../ja/taggers/whitelist.py | 40 + .../text_normalization/ja/taggers/word.py | 30 + .../text_normalization/ja/utils.py | 56 + .../ja/verbalizers/__init__.py | 13 + .../ja/verbalizers/cardinal.py | 52 + .../text_normalization/ja/verbalizers/date.py | 55 + .../ja/verbalizers/decimal.py | 52 + .../ja/verbalizers/fraction.py | 123 + .../ja/verbalizers/ordinal.py | 42 + .../ja/verbalizers/post_processing.py | 113 + .../ja/verbalizers/postprocessor.py | 81 + .../text_normalization/ja/verbalizers/time.py | 48 + .../ja/verbalizers/verbalize.py | 66 + .../ja/verbalizers/verbalize_final.py | 54 + .../ja/verbalizers/whitelist.py | 38 + .../text_normalization/ja/verbalizers/word.py | 31 + .../text_normalization/ko/__init__.py | 17 + .../text_normalization/ko/data/__init__.py | 13 + .../ko/data/date/__init__.py | 13 + .../ko/data/date/exceptions.tsv | 2 + .../text_normalization/ko/data/date/week.tsv | 8 + .../ko/data/electronic/__init__.py | 13 + .../ko/data/electronic/cc_cues.tsv | 11 + .../ko/data/electronic/domain.tsv | 28 + .../ko/data/electronic/extensions.tsv | 6 + .../ko/data/electronic/symbol.tsv | 12 + .../ko/data/measure/__init__.py | 13 + .../ko/data/measure/unit.tsv | 18 + .../ko/data/money/__init__.py | 13 + .../ko/data/money/currency_major.tsv | 22 + .../ko/data/number/__init__.py | 13 + .../ko/data/number/counter_suffix.tsv | 16 + .../ko/data/number/digit.tsv | 9 + .../ko/data/number/native_ones.tsv | 9 + .../ko/data/number/tens.tsv | 9 + .../ko/data/number/zero.tsv | 1 + .../ko/data/ordinal/__init__.py | 13 + .../ko/data/ordinal/digit.tsv | 8 + .../ko/data/ordinal/exceptions.tsv | 5 + .../ko/data/ordinal/tens.tsv | 2 + .../ko/data/ordinal/tens_prefix.tsv | 3 + .../ko/data/time/__init__.py | 13 + .../ko/data/time/division.tsv | 7 + .../text_normalization/ko/data/time/hour.tsv | 12 + .../text_normalization/ko/data/whitelist.tsv | 39 + .../text_normalization/ko/graph_utils.py | 173 + .../text_normalization/ko/taggers/__init__.py | 13 + .../text_normalization/ko/taggers/cardinal.py | 374 + .../text_normalization/ko/taggers/date.py | 323 + .../text_normalization/ko/taggers/decimal.py | 76 + .../ko/taggers/electronic.py | 168 + .../text_normalization/ko/taggers/fraction.py | 113 + .../text_normalization/ko/taggers/measure.py | 106 + .../text_normalization/ko/taggers/money.py | 99 + .../text_normalization/ko/taggers/ordinal.py | 85 + .../ko/taggers/punctuation.py | 37 + .../ko/taggers/telephone.py | 115 + .../text_normalization/ko/taggers/time.py | 192 + .../ko/taggers/tokenize_and_classify.py | 114 + .../ko/taggers/whitelist.py | 29 + .../text_normalization/ko/taggers/word.py | 36 + .../text_normalization/ko/utils.py | 60 + .../ko/verbalizers/__init__.py | 13 + .../ko/verbalizers/cardinal.py | 53 + .../text_normalization/ko/verbalizers/date.py | 55 + .../ko/verbalizers/decimal.py | 48 + .../ko/verbalizers/electronic.py | 141 + .../ko/verbalizers/fraction.py | 163 + .../ko/verbalizers/measure.py | 89 + .../ko/verbalizers/money.py | 110 + .../ko/verbalizers/ordinal.py | 42 + .../ko/verbalizers/telephone.py | 83 + .../text_normalization/ko/verbalizers/time.py | 63 + .../ko/verbalizers/verbalize.py | 74 + .../ko/verbalizers/verbalize_final.py | 72 + .../ko/verbalizers/whitelist.py | 39 + .../text_normalization/ko/verbalizers/word.py | 31 + .../text_normalization/normalize.py | 142 +- .../normalize_with_audio.py | 44 +- .../text_normalization/pt/__init__.py | 13 + .../text_normalization/pt/data/__init__.py | 13 + .../pt/data/date/__init__.py | 13 + .../pt/data/date/months.tsv | 21 + .../pt/data/date/numeric_separators.tsv | 3 + .../pt/data/date/verbal_phrases.tsv | 1 + .../pt/data/electronic/__init__.py | 13 + .../pt/data/electronic/domain.tsv | 7 + .../electronic/electronic_spoken_unit.tsv | 16 + .../pt/data/electronic/server_name.tsv | 6 + .../pt/data/electronic/symbols.tsv | 22 + .../pt/data/fractions/__init__.py | 13 + .../pt/data/fractions/ordinal_exceptions.tsv | 2 + .../pt/data/fractions/powers_of_ten.tsv | 1 + .../pt/data/fractions/specials.tsv | 5 + .../pt/data/measure/__init__.py | 13 + .../pt/data/measure/measurements_plural.tsv | 64 + .../pt/data/measure/measurements_singular.tsv | 65 + .../pt/data/money/__init__.py | 13 + .../pt/data/money/currency_major.tsv | 5 + .../pt/data/money/currency_major_plural.tsv | 5 + .../pt/data/money/currency_minor.tsv | 5 + .../pt/data/money/currency_minor_plural.tsv | 1 + .../pt/data/numbers/__init__.py | 13 + .../pt/data/numbers/cardinal_specials.tsv | 4 + .../numbers/decimal_fractional_specials.tsv | 3 + .../pt/data/numbers/decimal_specials.tsv | 2 + .../pt/data/numbers/digit.tsv | 9 + .../pt/data/numbers/hundreds.tsv | 8 + .../pt/data/numbers/quantity_words.tsv | 9 + .../pt/data/numbers/scales.tsv | 4 + .../pt/data/numbers/teens.tsv | 10 + .../pt/data/numbers/tens.tsv | 8 + .../pt/data/numbers/zero.tsv | 1 + .../pt/data/ordinals/__init__.py | 13 + .../pt/data/ordinals/digit.tsv | 10 + .../pt/data/ordinals/feminine.tsv | 11 + .../pt/data/ordinals/hundreds.tsv | 10 + .../pt/data/ordinals/specials.tsv | 2 + .../pt/data/ordinals/teen.tsv | 9 + .../pt/data/ordinals/ties.tsv | 8 + .../pt/data/telephone/__init__.py | 13 + .../pt/data/telephone/ip_prompt.tsv | 2 + .../pt/data/telephone/telephone_prompt.tsv | 4 + .../pt/data/time/__init__.py | 13 + .../pt/data/time/day_period_suffix.tsv | 4 + .../pt/data/whitelist/__init__.py | 13 + .../pt/data/whitelist/ipa_symbols.tsv | 521 ++ .../pt/data/whitelist/symbol.tsv | 23 + .../pt/data/whitelist/tts.tsv | 0 .../text_normalization/pt/graph_utils.py | 193 + .../text_normalization/pt/taggers/__init__.py | 13 + .../text_normalization/pt/taggers/cardinal.py | 303 + .../text_normalization/pt/taggers/date.py | 151 + .../text_normalization/pt/taggers/decimal.py | 91 + .../pt/taggers/electronic.py | 82 + .../text_normalization/pt/taggers/fraction.py | 117 + .../text_normalization/pt/taggers/measure.py | 78 + .../text_normalization/pt/taggers/money.py | 171 + .../text_normalization/pt/taggers/ordinal.py | 83 + .../pt/taggers/punctuation.py | 63 + .../pt/taggers/telephone.py | 119 + .../text_normalization/pt/taggers/time.py | 180 + .../pt/taggers/tokenize_and_classify.py | 135 + .../pt/taggers/whitelist.py | 111 + .../text_normalization/pt/taggers/word.py | 81 + .../text_normalization/pt/utils.py | 49 + .../pt/verbalizers/__init__.py | 13 + .../pt/verbalizers/cardinal.py | 68 + .../text_normalization/pt/verbalizers/date.py | 52 + .../pt/verbalizers/decimal.py | 67 + .../pt/verbalizers/electronic.py | 79 + .../pt/verbalizers/fraction.py | 109 + .../pt/verbalizers/measure.py | 98 + .../pt/verbalizers/money.py | 96 + .../pt/verbalizers/ordinal.py | 47 + .../pt/verbalizers/telephone.py | 57 + .../text_normalization/pt/verbalizers/time.py | 125 + .../pt/verbalizers/verbalize.py | 66 + .../pt/verbalizers/verbalize_final.py | 71 + .../text_normalization/pt/verbalizers/word.py | 32 + .../text_normalization/ru/alphabet.py | 1 + .../text_normalization/ru/taggers/cardinal.py | 5 +- .../text_normalization/ru/taggers/date.py | 7 +- .../text_normalization/ru/taggers/decimals.py | 5 +- .../ru/taggers/electronic.py | 3 +- .../text_normalization/ru/taggers/measure.py | 3 +- .../text_normalization/ru/taggers/money.py | 3 +- .../ru/taggers/number_names.py | 3 +- .../text_normalization/ru/taggers/ordinal.py | 5 +- .../ru/taggers/telephone.py | 15 +- .../text_normalization/ru/taggers/time.py | 5 +- .../ru/taggers/tokenize_and_classify.py | 10 +- .../ru/taggers/whitelist.py | 3 +- .../text_normalization/ru/taggers/word.py | 3 +- .../text_normalization/ru/utils.py | 11 +- .../ru/verbalizers/cardinal.py | 3 +- .../text_normalization/ru/verbalizers/date.py | 3 +- .../ru/verbalizers/decimal.py | 3 +- .../ru/verbalizers/electronic.py | 3 +- .../ru/verbalizers/measure.py | 5 +- .../ru/verbalizers/money.py | 3 +- .../ru/verbalizers/ordinal.py | 3 +- .../ru/verbalizers/telephone.py | 3 +- .../text_normalization/ru/verbalizers/time.py | 3 +- .../ru/verbalizers/verbalize_final.py | 10 +- .../text_normalization/run_evaluate.py | 11 +- .../text_normalization/rw/__init__.py | 18 + .../text_normalization/rw/data/__init__.py | 15 + .../rw/data/cardinal/__init__.py | 15 + .../rw/data/cardinal/digits.tsv | 9 + .../rw/data/cardinal/digits_for_thousands.tsv | 10 + .../cardinal/digits_millions_trillions.tsv | 10 + .../rw/data/cardinal/hundreds.tsv | 9 + .../rw/data/cardinal/hundreds_of_millions.tsv | 9 + .../data/cardinal/hundreds_of_thousands.tsv | 9 + .../data/cardinal/hundreds_of_trillions.tsv | 9 + .../rw/data/cardinal/millions.tsv | 9 + .../rw/data/cardinal/tens.tsv | 9 + .../rw/data/cardinal/tens_of_millions.tsv | 9 + .../rw/data/cardinal/tens_of_thousands.tsv | 9 + .../rw/data/cardinal/tens_of_trillions.tsv | 9 + .../rw/data/cardinal/thousands.tsv | 10 + .../rw/data/cardinal/trillions.tsv | 9 + .../rw/data/time/__init__.py | 15 + .../text_normalization/rw/data/time/hours.tsv | 12 + .../rw/data/time/minutes.tsv | 60 + .../rw/data/whitelist/__init__.py | 15 + .../data/whitelist/kinya_transliterations.tsv | 175 + .../text_normalization/rw/graph_utils.py | 297 + .../text_normalization/rw/taggers/__init__.py | 15 + .../text_normalization/rw/taggers/cardinal.py | 243 + .../text_normalization/rw/taggers/time.py | 43 + .../rw/taggers/tokenize_and_classify.py | 78 + .../rw/taggers/whitelist.py | 32 + .../text_normalization/rw/utils.py | 27 + .../rw/verbalizers/__init__.py | 15 + .../text_normalization/rw/verbalizers/time.py | 42 + .../rw/verbalizers/verbalize.py | 29 + .../rw/verbalizers/verbalize_final.py | 53 + .../se/data/electronic/symbols.tsv | 5 +- .../se/data/inflection/case_suffixes.tsv | 9 + .../data/inflection/ordinal_case_suffixes.tsv | 12 + .../se/data/measure/unit_genitive.tsv | 2 + .../se/data/measure/unit_rate.tsv | 4 + .../se/data/measure/unit_simple.tsv | 38 + .../se/data/money/currency_major.tsv | 12 + .../se/data/money/currency_major_gen.tsv | 3 + .../se/data/numbers/compound_digit.tsv | 8 + .../text_normalization/se/data/whitelist.tsv | 76 + .../text_normalization/se/graph_utils.py | 7 +- .../se/taggers/abbreviation.py | 10 +- .../text_normalization/se/taggers/cardinal.py | 67 +- .../text_normalization/se/taggers/date.py | 3 +- .../text_normalization/se/taggers/decimal.py | 17 +- .../se/taggers/electronic.py | 3 +- .../text_normalization/se/taggers/fraction.py | 3 +- .../text_normalization/se/taggers/measure.py | 233 +- .../text_normalization/se/taggers/money.py | 231 +- .../text_normalization/se/taggers/ordinal.py | 31 +- .../se/taggers/telephone.py | 7 +- .../text_normalization/se/taggers/time.py | 9 +- .../se/taggers/tokenize_and_classify.py | 25 +- .../se/taggers/whitelist.py | 3 +- .../text_normalization/se/taggers/word.py | 3 +- .../text_normalization/se/utils.py | 24 +- .../se/verbalizers/cardinal.py | 3 +- .../text_normalization/se/verbalizers/date.py | 3 +- .../se/verbalizers/decimals.py | 15 +- .../se/verbalizers/electronic.py | 3 +- .../se/verbalizers/fraction.py | 3 +- .../se/verbalizers/measure.py | 5 +- .../se/verbalizers/money.py | 3 +- .../se/verbalizers/ordinal.py | 3 +- .../se/verbalizers/telephone.py | 9 +- .../text_normalization/se/verbalizers/time.py | 3 +- .../se/verbalizers/verbalize_final.py | 3 +- .../text_normalization/sv/graph_utils.py | 3 +- .../sv/taggers/abbreviation.py | 3 +- .../text_normalization/sv/taggers/cardinal.py | 31 +- .../text_normalization/sv/taggers/date.py | 3 +- .../text_normalization/sv/taggers/decimal.py | 3 +- .../sv/taggers/electronic.py | 3 +- .../text_normalization/sv/taggers/fraction.py | 3 +- .../text_normalization/sv/taggers/measure.py | 7 +- .../text_normalization/sv/taggers/money.py | 3 +- .../text_normalization/sv/taggers/ordinal.py | 12 +- .../sv/taggers/telephone.py | 7 +- .../text_normalization/sv/taggers/time.py | 9 +- .../sv/taggers/tokenize_and_classify.py | 36 +- .../tokenize_and_classify_with_audio.py | 8 +- .../sv/taggers/whitelist.py | 3 +- .../text_normalization/sv/taggers/word.py | 3 +- .../sv/verbalizers/cardinal.py | 3 +- .../text_normalization/sv/verbalizers/date.py | 3 +- .../sv/verbalizers/decimals.py | 15 +- .../sv/verbalizers/electronic.py | 3 +- .../sv/verbalizers/fraction.py | 3 +- .../sv/verbalizers/measure.py | 3 +- .../sv/verbalizers/money.py | 3 +- .../sv/verbalizers/ordinal.py | 3 +- .../sv/verbalizers/telephone.py | 9 +- .../text_normalization/sv/verbalizers/time.py | 3 +- .../sv/verbalizers/verbalize_final.py | 9 +- .../text_normalization/token_parser.py | 12 +- .../text_normalization/utils_audio_based.py | 9 +- .../text_normalization/vi/__init__.py | 13 + .../text_normalization/vi/data/__init__.py | 13 + .../vi/data/date/__init__.py | 13 + .../text_normalization/vi/data/date/days.tsv | 40 + .../vi/data/date/months.tsv | 21 + .../vi/data/date/year_suffix.tsv | 4 + .../vi/data/fraction/__init__.py | 13 + .../data/fraction/denominator_exceptions.tsv | 1 + .../vi/data/measure/__init__.py | 13 + .../vi/data/measure/base_units.tsv | 20 + .../vi/data/measure/measurements_minimal.tsv | 33 + .../vi/data/measure/prefixes.tsv | 17 + .../vi/data/money/__init__.py | 13 + .../vi/data/money/currency.tsv | 51 + .../vi/data/money/currency_minor.tsv | 7 + .../vi/data/money/per_unit_bases.tsv | 8 + .../vi/data/money/per_unit_non_metric.tsv | 28 + .../vi/data/money/per_unit_prefixes.tsv | 6 + .../vi/data/numbers/__init__.py | 13 + .../vi/data/numbers/digit.tsv | 9 + .../vi/data/numbers/digit_special.tsv | 3 + .../vi/data/numbers/magnitudes.tsv | 8 + .../vi/data/numbers/quantity_abbr.tsv | 7 + .../vi/data/numbers/teen.tsv | 10 + .../vi/data/numbers/ties.tsv | 8 + .../vi/data/numbers/zero.tsv | 1 + .../vi/data/ordinal/__init__.py | 13 + .../vi/data/ordinal/ordinal_exceptions.tsv | 2 + .../vi/data/roman/__init__.py | 13 + .../vi/data/roman/key_word.tsv | 16 + .../vi/data/roman/roman_numerals.tsv | 13 + .../vi/data/time/__init__.py | 13 + .../vi/data/time/time_units.tsv | 4 + .../vi/data/time/time_zones.tsv | 18 + .../text_normalization/vi/data/whitelist.tsv | 0 .../text_normalization/vi/graph_utils.py | 165 + .../text_normalization/vi/taggers/__init__.py | 13 + .../text_normalization/vi/taggers/cardinal.py | 244 + .../text_normalization/vi/taggers/date.py | 168 + .../text_normalization/vi/taggers/decimal.py | 165 + .../text_normalization/vi/taggers/fraction.py | 72 + .../text_normalization/vi/taggers/measure.py | 153 + .../text_normalization/vi/taggers/money.py | 198 + .../text_normalization/vi/taggers/ordinal.py | 61 + .../vi/taggers/punctuation.py | 35 + .../text_normalization/vi/taggers/range.py | 66 + .../text_normalization/vi/taggers/roman.py | 91 + .../text_normalization/vi/taggers/time.py | 140 + .../vi/taggers/tokenize_and_classify.py | 170 + .../vi/taggers/whitelist.py | 70 + .../text_normalization/vi/taggers/word.py | 46 + .../text_normalization/vi/utils.py | 42 + .../vi/verbalizers/__init__.py | 13 + .../vi/verbalizers/cardinal.py | 55 + .../text_normalization/vi/verbalizers/date.py | 71 + .../vi/verbalizers/decimal.py | 103 + .../vi/verbalizers/fraction.py | 55 + .../vi/verbalizers/measure.py | 67 + .../vi/verbalizers/money.py | 143 + .../vi/verbalizers/ordinal.py | 48 + .../vi/verbalizers/post_processing.py | 139 + .../vi/verbalizers/range.py | 46 + .../vi/verbalizers/roman.py | 45 + .../text_normalization/vi/verbalizers/time.py | 174 + .../vi/verbalizers/verbalize.py | 85 + .../vi/verbalizers/verbalize_final.py | 72 + .../vi/verbalizers/whitelist.py | 48 + .../text_normalization/vi/verbalizers/word.py | 43 + .../zh/data/char/punctuations_zh.tsv | 2 + .../zh/data/math/symbol.tsv | 1 + .../zh/data/measure/units_en.tsv | 25 - .../zh/data/measure/units_zh.tsv | 211 - .../zh/data/money/currency_major.tsv | 1 - .../zh/data/number/digit_alt.tsv | 9 + .../zh/data/number/suffix.tsv | 23 + .../zh/data/number/teen.tsv | 10 + .../zh/data/number/teen_alt.tsv | 10 + .../zh/data/number/ties.tsv | 8 + .../text_normalization/zh/data/whitelist.tsv | 7 - .../text_normalization/zh/graph_utils.py | 36 +- .../text_normalization/zh/taggers/cardinal.py | 767 +- .../text_normalization/zh/taggers/date.py | 24 +- .../text_normalization/zh/taggers/decimal.py | 33 +- .../text_normalization/zh/taggers/fraction.py | 93 +- .../zh/taggers/math_symbol.py | 44 - .../text_normalization/zh/taggers/measure.py | 65 +- .../text_normalization/zh/taggers/money.py | 82 +- .../text_normalization/zh/taggers/ordinal.py | 9 +- .../zh/taggers/preprocessor.py | 27 +- .../zh/taggers/punctuation.py | 10 +- .../text_normalization/zh/taggers/time.py | 19 +- .../zh/taggers/tokenize_and_classify.py | 127 +- .../zh/taggers/whitelist.py | 25 +- .../text_normalization/zh/taggers/word.py | 63 +- .../text_normalization/zh/utils.py | 10 +- .../zh/verbalizers/cardinal.py | 8 +- .../text_normalization/zh/verbalizers/date.py | 23 +- .../zh/verbalizers/decimal.py | 22 +- .../zh/verbalizers/fraction.py | 18 +- .../zh/verbalizers/measure.py | 61 +- .../zh/verbalizers/money.py | 11 +- .../zh/verbalizers/ordinal.py | 9 +- .../zh/verbalizers/post_processing.py | 113 + .../zh/verbalizers/postprocessor.py | 21 +- .../text_normalization/zh/verbalizers/time.py | 13 +- .../zh/verbalizers/verbalize.py | 27 +- .../zh/verbalizers/verbalize_final.py | 21 +- .../zh/verbalizers/whitelist.py | 9 +- .../text_normalization/zh/verbalizers/word.py | 11 +- nemo_text_processing/utils/__init__.py | 15 + nemo_text_processing/utils/logging.py | 23 + requirements/requirements.txt | 4 +- requirements/requirements_test.txt | 8 +- setup.cfg | 2 +- setup.py | 19 +- tests/conftest.py | 16 +- .../test_cases_money.txt | 3 +- .../nemo_text_processing/ar/test_cardinal.py | 3 +- tests/nemo_text_processing/ar/test_decimal.py | 3 +- .../nemo_text_processing/ar/test_fraction.py | 3 +- tests/nemo_text_processing/ar/test_money.py | 7 +- .../ar/test_sparrowhawk_normalization.sh | 71 + .../nemo_text_processing/ar/test_whitelist.py | 3 +- .../test_audio_based_utils.py | 1 + .../test_cases_date.txt | 1 - .../test_cases_decimal.txt | 5 +- .../test_cases_electronic.txt | 14 +- .../test_cases_time.txt | 4 +- .../nemo_text_processing/de/test_cardinal.py | 3 +- tests/nemo_text_processing/de/test_date.py | 3 +- tests/nemo_text_processing/de/test_decimal.py | 3 +- .../de/test_electronic.py | 3 +- .../nemo_text_processing/de/test_fraction.py | 3 +- tests/nemo_text_processing/de/test_measure.py | 3 +- tests/nemo_text_processing/de/test_money.py | 3 +- .../de/test_normalization_with_audio.py | 3 +- tests/nemo_text_processing/de/test_ordinal.py | 3 +- ..._sparrowhawk_inverse_text_normalization.sh | 6 +- .../de/test_sparrowhawk_normalization.sh | 6 +- .../nemo_text_processing/de/test_telephone.py | 3 +- tests/nemo_text_processing/de/test_time.py | 3 +- .../nemo_text_processing/de/test_whitelist.py | 3 +- tests/nemo_text_processing/de/test_word.py | 3 +- .../test_cases_measure.txt | 2 +- .../test_cases_whitelist.txt | 1 - .../test_cases_address.txt | 2 + .../test_cases_electronic.txt | 15 +- .../test_cases_measure.txt | 2 + .../test_cases_money.txt | 8 +- .../test_cases_ordinal.txt | 2 +- .../test_cases_punctuation.txt | 3 +- .../test_cases_punctuation_match_input.txt | 2 +- .../test_cases_range.txt | 4 +- .../test_cases_serial.txt | 4 +- .../test_cases_time.txt | 2 + .../test_cases_whitelist.txt | 1 + .../test_cases_word.txt | 1 + tests/nemo_text_processing/en/test_address.py | 7 +- .../nemo_text_processing/en/test_cardinal.py | 7 +- tests/nemo_text_processing/en/test_date.py | 3 +- tests/nemo_text_processing/en/test_decimal.py | 7 +- .../en/test_electronic.py | 7 +- .../nemo_text_processing/en/test_fraction.py | 7 +- tests/nemo_text_processing/en/test_math.py | 7 +- tests/nemo_text_processing/en/test_measure.py | 7 +- tests/nemo_text_processing/en/test_money.py | 11 +- .../en/test_normalization_with_audio.py | 3 +- tests/nemo_text_processing/en/test_ordinal.py | 7 +- .../en/test_punctuation.py | 9 +- tests/nemo_text_processing/en/test_range.py | 7 +- tests/nemo_text_processing/en/test_roman.py | 7 +- tests/nemo_text_processing/en/test_serial.py | 7 +- ..._sparrowhawk_inverse_text_normalization.sh | 34 +- ...owhawk_inverse_text_normalization_cased.sh | 56 +- .../en/test_sparrowhawk_normalization.sh | 51 +- .../en/test_special_text.py | 7 +- .../nemo_text_processing/en/test_telephone.py | 3 +- .../en/test_text_split.py | 1 + tests/nemo_text_processing/en/test_time.py | 3 +- .../nemo_text_processing/en/test_whitelist.py | 3 +- tests/nemo_text_processing/en/test_word.py | 3 +- .../test_cases_cardinal_cased.txt | 30 + .../test_cases_date_cased.txt | 8 + .../test_cases_decimal_cased.txt | 6 + .../test_cases_electronic_cased.txt | 5 + .../test_cases_measure.txt | 2 +- .../test_cases_measure_cased.txt | 11 + .../test_cases_money_cased.txt | 6 + .../test_cases_ordinal_cased.txt | 11 + .../test_cases_telephone_cased.txt | 6 + .../test_cases_time.txt | 2 +- .../test_cases_time_cased.txt | 9 + .../test_cases_whitelist.txt | 10 +- .../test_cases_word_cased.txt | 11 + .../test_cases_electronic.txt | 9 +- .../test_cases_ordinal.txt | 6 +- .../test_cases_telephone.txt | 4 +- .../test_cases_whitelist.txt | 4 +- .../nemo_text_processing/es/test_cardinal.py | 21 +- tests/nemo_text_processing/es/test_date.py | 16 +- tests/nemo_text_processing/es/test_decimal.py | 16 +- .../es/test_electronic.py | 16 +- .../nemo_text_processing/es/test_fraction.py | 19 +- tests/nemo_text_processing/es/test_measure.py | 16 +- tests/nemo_text_processing/es/test_money.py | 16 +- .../es/test_normalization_with_audio.py | 3 +- tests/nemo_text_processing/es/test_ordinal.py | 20 +- ..._sparrowhawk_inverse_text_normalization.sh | 6 +- .../es/test_sparrowhawk_normalization.sh | 6 +- .../nemo_text_processing/es/test_telephone.py | 16 +- tests/nemo_text_processing/es/test_time.py | 16 +- .../nemo_text_processing/es/test_whitelist.py | 3 +- tests/nemo_text_processing/es/test_word.py | 16 +- .../test_cases_cardinal_cased.txt | 60 + .../test_cases_date.txt | 2 +- .../test_cases_date_cased.txt | 20 + .../test_cases_decimal_cased.txt | 19 + .../test_cases_electronic_cased.txt | 14 + .../test_cases_measure.txt | 2 +- .../test_cases_measure_cased.txt | 22 + .../test_cases_money_cased.txt | 16 + .../test_cases_ordinal_cased.txt | 21 + .../test_cases_telephone_cased.txt | 19 + .../test_cases_time.txt | 36 +- .../test_cases_time_cased.txt | 19 + .../test_cases_whitelist.txt | 10 +- .../test_cases_word_cased.txt | 17 + .../es_en/test_cardinal.py | 14 +- tests/nemo_text_processing/es_en/test_date.py | 13 +- .../es_en/test_decimal.py | 13 +- .../es_en/test_electronic.py | 17 +- .../es_en/test_fraction.py | 3 +- .../es_en/test_measure.py | 17 +- .../nemo_text_processing/es_en/test_money.py | 15 +- .../es_en/test_ordinal.py | 15 +- ..._sparrowhawk_inverse_text_normalization.sh | 6 +- .../es_en/test_telephone.py | 17 +- tests/nemo_text_processing/es_en/test_time.py | 17 +- .../es_en/test_whitelist.py | 3 +- tests/nemo_text_processing/es_en/test_word.py | 17 +- .../test_cases_date.txt | 13 + .../test_cases_whitelist.txt | 7 + .../test_cases_word.txt | 49 + .../nemo_text_processing/fr/test_cardinal.py | 3 +- tests/nemo_text_processing/fr/test_date.py | 13 +- tests/nemo_text_processing/fr/test_decimal.py | 3 +- .../fr/test_electronic.py | 3 +- .../nemo_text_processing/fr/test_fraction.py | 3 +- tests/nemo_text_processing/fr/test_measure.py | 3 +- tests/nemo_text_processing/fr/test_money.py | 3 +- tests/nemo_text_processing/fr/test_ordinal.py | 3 +- ..._sparrowhawk_inverse_text_normalization.sh | 10 +- .../fr/test_sparrowhawk_normalization.sh | 21 +- .../nemo_text_processing/fr/test_telephone.py | 3 +- tests/nemo_text_processing/fr/test_time.py | 3 +- .../nemo_text_processing/fr/test_whitelist.py | 13 +- tests/nemo_text_processing/fr/test_word.py | 13 +- tests/nemo_text_processing/he/__init__.py | 13 + .../test_cases_cardinal.txt | 138 + .../test_cases_date.txt | 29 + .../test_cases_decimal.txt | 66 + .../test_cases_measure.txt | 10 + .../test_cases_time.txt | 34 + .../test_cases_whitelist.txt | 4 + .../test_full_sentences.txt | 56 + .../nemo_text_processing/he/test_cardinal.py | 31 + tests/nemo_text_processing/he/test_date.py | 31 + tests/nemo_text_processing/he/test_decimal.py | 31 + .../he/test_full_sentences.py | 31 + tests/nemo_text_processing/he/test_measure.py | 31 + ..._sparrowhawk_inverse_text_normalization.sh | 61 + tests/nemo_text_processing/he/test_time.py | 31 + .../nemo_text_processing/he/test_whitelist.py | 31 + tests/nemo_text_processing/hi/__init__.py | 13 + .../test_cases_address.txt | 25 + .../test_cases_cardinal.txt | 54 + .../test_cases_date.txt | 42 + .../test_cases_decimal.txt | 13 + .../test_cases_fraction.txt | 31 + .../test_cases_measure.txt | 48 + .../test_cases_money.txt | 50 + .../test_cases_ordinal.txt | 13 + .../test_cases_telephone.txt | 28 + .../test_cases_time.txt | 25 + .../test_cases_whitelist.txt | 8 + .../test_cases_word.txt | 15 + .../test_cases_address.txt | 52 + .../test_cases_cardinal.txt | 161 + .../test_cases_date.txt | 39 + .../test_cases_decimal.txt | 24 + .../test_cases_electronic.txt | 65 + .../test_cases_fraction.txt | 27 + .../test_cases_measure.txt | 69 + .../test_cases_money.txt | 144 + .../test_cases_ordinal.txt | 72 + .../test_cases_roman.txt | 23 + .../test_cases_serial.txt | 31 + .../test_cases_telephone.txt | 25 + .../test_cases_time.txt | 18 + .../test_cases_whitelist.txt | 14 + .../test_cases_word.txt | 20 + tests/nemo_text_processing/hi/test_address.py | 42 + .../nemo_text_processing/hi/test_cardinal.py | 42 + tests/nemo_text_processing/hi/test_date.py | 42 + tests/nemo_text_processing/hi/test_decimal.py | 42 + .../hi/test_electronic.py | 33 + .../nemo_text_processing/hi/test_fraction.py | 42 + tests/nemo_text_processing/hi/test_measure.py | 42 + tests/nemo_text_processing/hi/test_money.py | 42 + tests/nemo_text_processing/hi/test_ordinal.py | 43 + tests/nemo_text_processing/hi/test_roman.py | 36 + .../test_preprocess.py => hi/test_serial.py} | 63 +- ..._sparrowhawk_inverse_text_normalization.sh | 88 + .../hi/test_sparrowhawk_normalization.sh | 121 + .../nemo_text_processing/hi/test_telephone.py | 31 + tests/nemo_text_processing/hi/test_time.py | 42 + .../nemo_text_processing/hi/test_whitelist.py | 42 + tests/nemo_text_processing/hi/test_word.py | 42 + tests/nemo_text_processing/hi_en/__init__.py | 13 + .../test_cases_address.txt | 41 + .../test_cases_cardinal.txt | 61 + .../test_cases_date.txt | 37 + .../test_cases_date_cased.txt | 70 + .../test_cases_decimal.txt | 25 + .../test_cases_electronic.txt | 17 + .../test_cases_fraction.txt | 39 + .../test_cases_measure.txt | 25 + .../test_cases_money.txt | 41 + .../test_cases_ordinal.txt | 26 + .../test_cases_telephone.txt | 39 + .../test_cases_time.txt | 29 + .../test_cases_whitelist.txt | 17 + .../test_cases_word.txt | 19 + .../hi_en/test_address.py | 38 + .../hi_en/test_cardinal.py | 48 + tests/nemo_text_processing/hi_en/test_date.py | 48 + .../hi_en/test_decimal.py | 48 + .../hi_en/test_electronic.py | 41 + .../hi_en/test_fraction.py | 38 + .../hi_en/test_measure.py | 48 + .../nemo_text_processing/hi_en/test_money.py | 48 + .../hi_en/test_ordinal.py | 48 + ..._sparrowhawk_inverse_text_normalization.sh | 102 + .../hi_en/test_telephone.py | 41 + tests/nemo_text_processing/hi_en/test_time.py | 48 + .../hi_en/test_whitelist.py | 48 + tests/nemo_text_processing/hi_en/test_word.py | 48 + .../test_cases_electronic.txt | 6 + .../nemo_text_processing/hu/test_cardinal.py | 3 +- tests/nemo_text_processing/hu/test_date.py | 3 +- tests/nemo_text_processing/hu/test_decimal.py | 3 +- .../hu/test_electronic.py | 3 +- .../nemo_text_processing/hu/test_fraction.py | 3 +- tests/nemo_text_processing/hu/test_measure.py | 3 +- tests/nemo_text_processing/hu/test_money.py | 3 +- tests/nemo_text_processing/hu/test_ordinal.py | 3 +- .../hu/test_sparrowhawk_normalization.sh | 6 +- .../nemo_text_processing/hu/test_telephone.py | 3 +- tests/nemo_text_processing/hu/test_time.py | 3 +- .../nemo_text_processing/hu/test_whitelist.py | 3 +- tests/nemo_text_processing/hu/test_word.py | 3 +- tests/nemo_text_processing/hy/__init__.py | 13 + .../test_cases_cardinal.txt | 23 + .../test_cases_decimal.txt | 14 + .../test_cases_fraction.txt | 24 + .../test_cases_measure.txt | 12 + .../test_cases_money.txt | 10 + .../test_cases_ordinal.txt | 11 + .../test_cases_time.txt | 9 + .../test_cases_whitelist.txt | 7 + .../test_cases_word.txt | 50 + .../test_cases_cardinal.txt | 12 + .../test_cases_decimal.txt | 9 + .../test_cases_fraction.txt | 10 + .../test_cases_measure.txt | 9 + .../test_cases_money.txt | 10 + .../test_cases_ordinal.txt | 11 + .../test_cases_time.txt | 9 + .../test_cases_whitelist.txt | 14 + .../test_cases_word.txt | 50 + .../nemo_text_processing/hy/test_cardinal.py | 42 + tests/nemo_text_processing/hy/test_decimal.py | 42 + .../nemo_text_processing/hy/test_fraction.py | 42 + tests/nemo_text_processing/hy/test_measure.py | 42 + tests/nemo_text_processing/hy/test_money.py | 42 + tests/nemo_text_processing/hy/test_ordinal.py | 42 + ..._sparrowhawk_inverse_text_normalization.sh | 71 + .../hy/test_sparrowhawk_normalization.sh | 69 + tests/nemo_text_processing/hy/test_time.py | 42 + .../nemo_text_processing/hy/test_whitelist.py | 42 + tests/nemo_text_processing/hy/test_word.py | 42 + .../test_cases_cardinal.txt | 4 +- .../test_cases_electronic.txt | 9 +- .../test_cases_measure.txt | 3 +- .../test_cases_money.txt | 3 +- .../test_cases_time.txt | 6 + .../test_cases_whitelist.txt | 3 +- .../nemo_text_processing/it/test_cardinal.py | 3 +- tests/nemo_text_processing/it/test_decimal.py | 3 +- .../it/test_electronic.py | 3 +- tests/nemo_text_processing/it/test_measure.py | 3 +- tests/nemo_text_processing/it/test_money.py | 3 +- .../it/test_sparrowhawk_normalization.sh | 23 +- .../{zh/test_math.py => it/test_time.py} | 17 +- .../nemo_text_processing/it/test_whitelist.py | 3 +- tests/nemo_text_processing/ja/__init__.py | 13 + .../test_cases_cardinal.txt | 28 + .../test_cases_date.txt | 31 + .../test_cases_decimal.txt | 32 + .../test_cases_fraction.txt | 34 + .../test_cases_ordinal.txt | 65 + .../test_cases_time.txt | 40 + .../test_cases_cardinal.txt | 43 + .../test_cases_date.txt | 200 + .../test_cases_decimal.txt | 15 + .../test_cases_fraction.txt | 18 + .../test_cases_ordinal.txt | 64 + .../test_cases_time.txt | 202 + .../nemo_text_processing/ja/test_cardinal.py | 42 + tests/nemo_text_processing/ja/test_date.py | 42 + tests/nemo_text_processing/ja/test_decimal.py | 41 + .../nemo_text_processing/ja/test_fraction.py | 42 + tests/nemo_text_processing/ja/test_ordinal.py | 41 + ..._sparrowhawk_inverse_text_normalization.sh | 55 + .../ja/test_sparrowhawk_normalization.sh | 53 + tests/nemo_text_processing/ja/test_time.py | 42 + tests/nemo_text_processing/ko/__init__.py | 13 + .../test_cases_cardinal.txt | 37 + .../test_cases_date.txt | 15 + .../test_cases_decimal.txt | 14 + .../test_cases_fraction.txt | 29 + .../test_cases_measure.txt | 46 + .../test_cases_money.txt | 29 + .../test_cases_ordinal.txt | 17 + .../test_cases_telephone.txt | 7 + .../test_cases_time.txt | 15 + .../test_cases_cardinal.txt | 68 + .../test_cases_date.txt | 29 + .../test_cases_decimal.txt | 28 + .../test_cases_electronic.txt | 20 + .../test_cases_fraction.txt | 28 + .../test_cases_measure.txt | 40 + .../test_cases_money.txt | 64 + .../test_cases_ordinal.txt | 54 + .../test_cases_telephone.txt | 39 + .../test_cases_time.txt | 25 + .../nemo_text_processing/ko/test_cardinal.py | 41 + tests/nemo_text_processing/ko/test_date.py | 41 + tests/nemo_text_processing/ko/test_decimal.py | 41 + .../ko/test_electronic.py | 32 + .../nemo_text_processing/ko/test_fraction.py | 41 + tests/nemo_text_processing/ko/test_measure.py | 41 + tests/nemo_text_processing/ko/test_money.py | 41 + tests/nemo_text_processing/ko/test_ordinal.py | 43 + ..._sparrowhawk_inverse_text_normalization.sh | 74 + .../ko/test_sparrowhawk_normalization.sh | 83 + .../nemo_text_processing/ko/test_telephone.py | 41 + tests/nemo_text_processing/ko/test_time.py | 41 + tests/nemo_text_processing/mr/__init__.py | 13 + .../test_cases_cardinal.txt | 33 + .../test_cases_date.txt | 29 + .../test_cases_decimal.txt | 29 + .../test_cases_time.txt | 29 + .../test_cases_word.txt | 11 + .../nemo_text_processing/mr/test_cardinal.py | 33 + tests/nemo_text_processing/mr/test_date.py | 32 + tests/nemo_text_processing/mr/test_decimal.py | 31 + ..._sparrowhawk_inverse_text_normalization.sh | 49 + tests/nemo_text_processing/mr/test_time.py | 31 + tests/nemo_text_processing/mr/test_word.py | 32 + .../test_cases_cardinal.txt | 121 + .../test_cases_date.txt | 26 + .../test_cases_decimal.txt | 58 + .../test_cases_electronic.txt | 13 + .../test_cases_fraction.txt | 22 + .../test_cases_measure.txt | 18 + .../test_cases_money.txt | 25 + .../test_cases_ordinal.txt | 39 + .../test_cases_telephone.txt | 7 + .../test_cases_time.txt | 30 + .../nemo_text_processing/pt/test_cardinal.py | 13 +- tests/nemo_text_processing/pt/test_date.py | 13 +- tests/nemo_text_processing/pt/test_decimal.py | 13 +- .../pt/test_electronic.py | 13 +- .../nemo_text_processing/pt/test_fraction.py | 32 + tests/nemo_text_processing/pt/test_measure.py | 16 +- tests/nemo_text_processing/pt/test_money.py | 16 +- tests/nemo_text_processing/pt/test_ordinal.py | 12 +- ..._sparrowhawk_inverse_text_normalization.sh | 6 +- .../pt/test_sparrowhawk_normalization.sh | 91 + .../nemo_text_processing/pt/test_telephone.py | 13 +- tests/nemo_text_processing/pt/test_time.py | 12 +- .../nemo_text_processing/pt/test_whitelist.py | 3 +- tests/nemo_text_processing/pt/test_word.py | 3 +- ..._sparrowhawk_inverse_text_normalization.sh | 6 +- tests/nemo_text_processing/rw/__init__.py | 15 + .../test_cases_cardinal.txt | 57 + .../test_cases_time.txt | 14 + .../test_cases_whitelist.txt | 3 + .../test_cases_word.txt | 26 + .../nemo_text_processing/rw/test_cardinal.py | 37 + .../rw/test_sparrowhawk_normalization.sh | 60 + tests/nemo_text_processing/rw/test_time.py | 34 + .../nemo_text_processing/rw/test_whitelist.py | 35 + tests/nemo_text_processing/rw/test_word.py | 35 + .../test_cases_abbreviation.txt | 5 + .../test_cases_cardinal.txt | 8 + .../test_cases_measure.txt | 10 + .../test_cases_money.txt | 8 + .../test_cases_ordinal.txt | 9 + .../test_cases_whitelist.txt | 5 + .../se/test_abbreviation.py | 31 + .../nemo_text_processing/se/test_cardinal.py | 3 +- tests/nemo_text_processing/se/test_date.py | 3 +- .../se/test_electronic.py | 3 +- tests/nemo_text_processing/se/test_measure.py | 32 + tests/nemo_text_processing/se/test_money.py | 32 + tests/nemo_text_processing/se/test_ordinal.py | 3 +- tests/nemo_text_processing/se/test_time.py | 3 +- .../nemo_text_processing/se/test_whitelist.py | 3 +- tests/nemo_text_processing/se/test_word.py | 3 +- .../nemo_text_processing/sv/test_cardinal.py | 3 +- tests/nemo_text_processing/sv/test_date.py | 3 +- tests/nemo_text_processing/sv/test_decimal.py | 3 +- .../sv/test_electronic.py | 3 +- .../nemo_text_processing/sv/test_fraction.py | 3 +- tests/nemo_text_processing/sv/test_measure.py | 3 +- tests/nemo_text_processing/sv/test_money.py | 3 +- .../sv/test_normalization_with_audio.py | 3 +- tests/nemo_text_processing/sv/test_ordinal.py | 3 +- ..._sparrowhawk_inverse_text_normalization.sh | 6 +- .../sv/test_sparrowhawk_normalization.sh | 6 +- .../nemo_text_processing/sv/test_telephone.py | 3 +- tests/nemo_text_processing/sv/test_time.py | 3 +- .../nemo_text_processing/sv/test_whitelist.py | 3 +- tests/nemo_text_processing/sv/test_word.py | 3 +- tests/nemo_text_processing/utils.py | 4 +- .../test_cases_cardinal.txt | 51 +- .../test_cases_date.txt | 55 +- .../test_cases_decimal.txt | 51 +- .../test_cases_electronic.txt | 10 +- .../test_cases_measure.txt | 19 +- .../test_cases_money.txt | 8 +- .../test_cases_telephone.txt | 27 +- .../test_cases_time.txt | 11 +- .../test_cases_cardinal.txt | 109 + .../test_cases_date.txt | 13 + .../test_cases_decimal.txt | 29 + .../test_cases_fraction.txt | 13 + .../test_cases_measure.txt | 63 + .../test_cases_money.txt | 30 + .../test_cases_ordinal.txt | 30 + .../test_cases_range.txt | 11 + .../test_cases_roman.txt | 57 + .../test_cases_time.txt | 117 + .../nemo_text_processing/vi/test_cardinal.py | 44 +- tests/nemo_text_processing/vi/test_date.py | 45 +- tests/nemo_text_processing/vi/test_decimal.py | 45 +- .../nemo_text_processing/vi/test_fraction.py | 46 +- tests/nemo_text_processing/vi/test_measure.py | 16 + tests/nemo_text_processing/vi/test_money.py | 49 +- tests/nemo_text_processing/vi/test_ordinal.py | 46 +- tests/nemo_text_processing/vi/test_range.py | 30 + tests/nemo_text_processing/vi/test_roman.py | 48 + ..._sparrowhawk_inverse_text_normalization.sh | 6 +- .../vi/test_sparrowhawk_normalization.sh | 87 + tests/nemo_text_processing/vi/test_time.py | 47 +- .../test_cases_whitelist.txt | 21 + .../test_cases_word.txt | 21 + .../test_cases_cardinal.txt | 158 +- .../test_cases_date.txt | 7 +- .../test_cases_decimal.txt | 15 +- .../test_cases_fraction.txt | 20 +- .../test_cases_math.txt | 2 - .../test_cases_measure.txt | 6 + .../test_cases_money.txt | 10 +- .../test_cases_ordinal.txt | 145 +- .../test_cases_preprocess.txt | 1 - .../test_cases_time.txt | 8 +- .../test_cases_whitelist.txt | 10 + .../test_cases_word.txt | 25 +- .../nemo_text_processing/zh/test_cardinal.py | 3 +- tests/nemo_text_processing/zh/test_date.py | 3 +- tests/nemo_text_processing/zh/test_decimal.py | 3 +- .../nemo_text_processing/zh/test_fraction.py | 3 +- tests/nemo_text_processing/zh/test_measure.py | 3 +- tests/nemo_text_processing/zh/test_money.py | 3 +- tests/nemo_text_processing/zh/test_ordinal.py | 3 +- ..._sparrowhawk_inverse_text_normalization.sh | 48 +- .../zh/test_sparrowhawk_normalization.sh | 52 +- tests/nemo_text_processing/zh/test_time.py | 17 +- .../nemo_text_processing/zh/test_whitelist.py | 41 + tests/nemo_text_processing/zh/test_word.py | 20 +- tools/text_processing_deployment/Dockerfile | 14 +- .../docker/launch.sh | 24 +- .../export_grammars.sh | 52 +- .../pynini_export.py | 137 +- tools/text_processing_deployment/sh_test.sh | 66 + tutorials/Text_(Inverse)_Normalization.ipynb | 24 +- tutorials/WFST_Tutorial.ipynb | 1784 ++-- 1849 files changed, 66596 insertions(+), 5834 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/dates/months_cased.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/dates/year_suffix_cased.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural_capitalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular_capitalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/ordinals/digit_capitalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/ordinals/hundreds_capitalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/ordinals/teen_capitalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/ordinals/ties_capitalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/ordinals/twenties_capitalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/time/time_suffix_cased.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/es/data/time/time_zone_cased.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/decimals/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/decimals/minutes_exception.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/measurements.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/months.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/months_name2number.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/months_ordinal2number.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/decimal_fractions.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/hundreds_exception.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/millions_exception.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/teen.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands_exception.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/viable_hours.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/numbers/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/ordinals/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/ordinals/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/prefix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/spaced_measurements.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/day_suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/evening_suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_evening.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_night.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_noon.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/midnight_to_hour.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/minute_to.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/minute_to_verbose.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/minute_verbose.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/night_suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/noon_suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/time_suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/time/to_hour.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/data/whitelist.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/he/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/punctuation.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/he/verbalizers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/address/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/address/cities.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/address/context_cues.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/address/digit_passthrough.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/address/ordinals.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/address/special_characters.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/address/states.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/date/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/date/century.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/date/months.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/measure/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/measure/measurements.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/money/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/money/currency.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/hundred.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/labels_exception.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/paune.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/teens_and_ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/thousands.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/numbers/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit_fem.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit_fem.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/morph_features.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix_fem.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties_fem.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/telephone/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/telephone/context_cues.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/telephone/country_codes.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/telephone/teens_and_ties_eng_to_hin.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/time/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/time/hour_for_paune.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/whitelist/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/data/whitelist/whitelist.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hi/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/punctuation.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/telephone.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi/verbalizers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/currency.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/measurement_dates.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/measurements.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/numbers/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/numbers/digits_no_one.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/numbers/ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/ordinals/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/ordinals/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/time/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/time/hours.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/time/minute_to.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/time/minutes.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/time/minutes_to.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/time/to_hour.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/data/whitelist.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/punctuation.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/hy/verbalizers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/char/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/char/charset_extension.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/char/charset_national_standard_2013_8105.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/char/fullwidth_to_halfwidth.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/char/oov_tags.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/char/punctuations_zh.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/char/upper_to_lower.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/date.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/day.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/months.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/numbers/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/numbers/teen.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/numbers/ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/time_minutes_seconds.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/data/whitelist.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ja/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/punctuation.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/post_processing.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/postprocessor.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py rename nemo_text_processing/{text_normalization/zh/verbalizers/math_symbol.py => inverse_text_normalization/ja/verbalizers/word.py} (77%) create mode 100644 nemo_text_processing/inverse_text_normalization/ko/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/currency.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/measure_units.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/months.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/numbers/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/numbers/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/ordinals/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/ordinals/counter_suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/ordinals/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/time/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/time/ten_prefix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/time/time_hours.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/data/whitelist.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/ko/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/telephone.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/measure.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/money.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py create mode 100644 nemo_text_processing/inverse_text_normalization/ko/verbalizers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/date/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/date/dates.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/date/months.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/date/prefixes.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/numbers/digits.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/numbers/hundred.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/numbers/tens.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/numbers/thousands.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/numbers/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/time/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/time/hours.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/time/hours_to.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/time/minutes.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/data/time/minutes_to.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/mr/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/punctuation.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/date.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/decimal.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/time.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/mr/verbalizers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/vi/data/electronic/url_symbols.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/vi/data/numbers/digit_special.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/vi/data/time/zero_prefix.tsv create mode 100644 nemo_text_processing/text_normalization/en/data/electronic/cc_cues.tsv create mode 100644 nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/fr/data/dates/__init__.py create mode 100644 nemo_text_processing/text_normalization/fr/data/dates/eras.tsv create mode 100644 nemo_text_processing/text_normalization/fr/data/dates/months.tsv rename nemo_text_processing/text_normalization/{it/data/__init__ .py => fr/data/measures/__init__.py} (100%) create mode 100644 nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv create mode 100644 nemo_text_processing/text_normalization/fr/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/fr/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/hi/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/address/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/address/cities.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/address/context.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/address/letters.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/address/special_characters.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/address/states.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/date/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/date/days.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/date/months.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/date/prefixes.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/date/suffixes.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/date/year_suffix.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/domain.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/elements.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/file_extensions.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/protocols.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/subscript_digit.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/symbol_classes.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/electronic/symbols.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/fraction/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/fraction/common_fractions.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/measure/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_list.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_map.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/measure/unit.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/measure/unit_year_formal.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/money/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/money/currency.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/money/currency_singular.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/money/major_minor_currencies.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/numbers/hundred.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties_en.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/numbers/thousands.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/numbers/zero.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/ordinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/ordinal/en_to_hi_digit.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/ordinal/exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/ordinal/suffixes.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/ordinal/suffixes_map.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/roman/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/roman/roman_ordinal_exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/serial/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/serial/chars.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/serial/power_special.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/serial/special_symbols.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/telephone/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/telephone/credit_context.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/telephone/landline_context.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/telephone/mobile_context.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/telephone/number.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/telephone/pincode_context.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/time/hours.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/time/minutes.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/time/seconds.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/whitelist/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/data/whitelist/abbreviations.tsv create mode 100644 nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv create mode 100644 nemo_text_processing/text_normalization/hi/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/decimal.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/electronic.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/fraction.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/money.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/punctuation.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/roman.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/serial.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/telephone.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/hi/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/hi/utils.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/decimal.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/electronic.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/fraction.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/money.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/post_processing.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/roman.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/telephone.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/hi/verbalizers/word.py create mode 100644 nemo_text_processing/text_normalization/hy/__init__.py create mode 100644 nemo_text_processing/text_normalization/hy/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/hy/data/currency.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/measurement_dates.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/measurements.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/hy/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/numbers/quantities.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/numbers/ties.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/ordinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/hy/data/ordinal/digit.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/hy/data/time/hours.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/time/minutes.tsv create mode 100644 nemo_text_processing/text_normalization/hy/data/whitelist.tsv create mode 100644 nemo_text_processing/text_normalization/hy/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/decimal.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/fraction.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/money.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/punctuation.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/hy/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/hy/utils.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/decimal.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/fraction.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/money.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/hy/verbalizers/word.py rename nemo_text_processing/text_normalization/it/data/{electronic/__init__ .py => __init__.py} (100%) rename nemo_text_processing/text_normalization/it/data/{numbers/__init__ .py => electronic/__init__.py} (100%) rename nemo_text_processing/text_normalization/it/{taggers/__init__ .py => data/numbers/__init__.py} (100%) create mode 100644 nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv rename nemo_text_processing/text_normalization/it/{verbalizers/__init__ .py => data/whitelist/__init__.py} (100%) rename nemo_text_processing/text_normalization/it/data/{ => whitelist}/whitelist.tsv (100%) create mode 100644 nemo_text_processing/text_normalization/it/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/it/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/it/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/it/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/ja/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/date/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/date/day.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/date/era.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/date/month.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/date/week.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/symbol.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/data/time/division.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/time/hour.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/time/minute.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/time/second.tsv create mode 100644 nemo_text_processing/text_normalization/ja/data/whitelist.tsv create mode 100644 nemo_text_processing/text_normalization/ja/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/decimal.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/fraction.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/punctuation.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/ja/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/ja/utils.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/decimal.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/fraction.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/ja/verbalizers/word.py create mode 100644 nemo_text_processing/text_normalization/ko/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/date/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/date/week.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/electronic/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/measure/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/measure/unit.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/money/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/money/currency_major.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/number/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/number/counter_suffix.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/number/digit.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/number/native_ones.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/number/tens.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/number/zero.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/ordinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/ordinal/digit.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/ordinal/exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/ordinal/tens.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/ordinal/tens_prefix.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/data/time/division.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/time/hour.tsv create mode 100644 nemo_text_processing/text_normalization/ko/data/whitelist.tsv create mode 100644 nemo_text_processing/text_normalization/ko/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/decimal.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/electronic.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/fraction.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/money.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/punctuation.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/telephone.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/ko/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/ko/utils.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/decimal.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/electronic.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/fraction.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/money.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/telephone.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/word.py create mode 100644 nemo_text_processing/text_normalization/pt/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/date/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/date/months.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/date/numeric_separators.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/date/verbal_phrases.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/electronic/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/electronic/domain.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/electronic/electronic_spoken_unit.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/electronic/server_name.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/electronic/symbols.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/fractions/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/fractions/ordinal_exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/fractions/powers_of_ten.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/fractions/specials.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/measure/__init__.py create mode 100755 nemo_text_processing/text_normalization/pt/data/measure/measurements_plural.tsv create mode 100755 nemo_text_processing/text_normalization/pt/data/measure/measurements_singular.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/money/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/money/currency_major.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/money/currency_major_plural.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/money/currency_minor.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/money/currency_minor_plural.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/cardinal_specials.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/decimal_fractional_specials.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/decimal_specials.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/hundreds.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/quantity_words.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/scales.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/teens.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/tens.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/numbers/zero.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/ordinals/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/ordinals/digit.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/ordinals/feminine.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/ordinals/hundreds.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/ordinals/specials.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/ordinals/teen.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/ordinals/ties.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/telephone/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/telephone/ip_prompt.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/telephone/telephone_prompt.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/time/day_period_suffix.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/whitelist/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/data/whitelist/ipa_symbols.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/whitelist/symbol.tsv create mode 100644 nemo_text_processing/text_normalization/pt/data/whitelist/tts.tsv create mode 100644 nemo_text_processing/text_normalization/pt/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/decimal.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/electronic.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/fraction.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/money.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/punctuation.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/telephone.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/pt/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/pt/utils.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/decimal.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/electronic.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/fraction.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/money.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/telephone.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/pt/verbalizers/word.py create mode 100644 nemo_text_processing/text_normalization/rw/__init__.py create mode 100644 nemo_text_processing/text_normalization/rw/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/digits.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/digits_for_thousands.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/digits_millions_trillions.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/hundreds.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_millions.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_thousands.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_trillions.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/millions.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/tens.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_millions.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_thousands.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_trillions.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/thousands.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/cardinal/trillions.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/rw/data/time/hours.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/time/minutes.tsv create mode 100644 nemo_text_processing/text_normalization/rw/data/whitelist/__init__.py create mode 100644 nemo_text_processing/text_normalization/rw/data/whitelist/kinya_transliterations.tsv create mode 100644 nemo_text_processing/text_normalization/rw/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/rw/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/rw/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/rw/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/rw/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/rw/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/rw/utils.py create mode 100644 nemo_text_processing/text_normalization/rw/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/rw/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/rw/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/rw/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/se/data/inflection/case_suffixes.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/inflection/ordinal_case_suffixes.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/measure/unit_genitive.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/measure/unit_rate.tsv create mode 100644 nemo_text_processing/text_normalization/se/data/numbers/compound_digit.tsv create mode 100644 nemo_text_processing/text_normalization/vi/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/date/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/date/days.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/date/months.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/date/year_suffix.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/fraction/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/fraction/denominator_exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/measure/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/measure/base_units.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/measure/measurements_minimal.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/measure/prefixes.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/money/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/money/currency.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/money/currency_minor.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/money/per_unit_bases.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/money/per_unit_non_metric.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/money/per_unit_prefixes.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/quantity_abbr.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/numbers/zero.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/ordinal/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/ordinal/ordinal_exceptions.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/roman/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/roman/key_word.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/roman/roman_numerals.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/time/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/data/time/time_units.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/time/time_zones.tsv create mode 100644 nemo_text_processing/text_normalization/vi/data/whitelist.tsv create mode 100644 nemo_text_processing/text_normalization/vi/graph_utils.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/date.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/decimal.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/fraction.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/measure.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/money.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/punctuation.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/range.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/roman.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/time.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/vi/taggers/word.py create mode 100644 nemo_text_processing/text_normalization/vi/utils.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/__init__.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/cardinal.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/date.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/decimal.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/fraction.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/measure.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/money.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/ordinal.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/post_processing.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/range.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/roman.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/time.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/verbalize.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/whitelist.py create mode 100644 nemo_text_processing/text_normalization/vi/verbalizers/word.py delete mode 100644 nemo_text_processing/text_normalization/zh/data/measure/units_zh.tsv create mode 100644 nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv create mode 100644 nemo_text_processing/text_normalization/zh/data/number/suffix.tsv create mode 100644 nemo_text_processing/text_normalization/zh/data/number/teen.tsv create mode 100644 nemo_text_processing/text_normalization/zh/data/number/teen_alt.tsv create mode 100644 nemo_text_processing/text_normalization/zh/data/number/ties.tsv delete mode 100644 nemo_text_processing/text_normalization/zh/taggers/math_symbol.py create mode 100644 nemo_text_processing/text_normalization/zh/verbalizers/post_processing.py create mode 100644 nemo_text_processing/utils/__init__.py create mode 100644 nemo_text_processing/utils/logging.py create mode 100755 tests/nemo_text_processing/ar/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_cardinal_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_date_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_decimal_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_electronic_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_money_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_ordinal_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_telephone_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time_cased.txt create mode 100644 tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_word_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_cardinal_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_decimal_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_electronic_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_money_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_ordinal_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_telephone_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time_cased.txt create mode 100644 tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_word_cased.txt create mode 100644 tests/nemo_text_processing/fr/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/fr/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/fr/data_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/he/__init__.py create mode 100644 tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/he/data_inverse_text_normalization/test_full_sentences.txt create mode 100644 tests/nemo_text_processing/he/test_cardinal.py create mode 100644 tests/nemo_text_processing/he/test_date.py create mode 100644 tests/nemo_text_processing/he/test_decimal.py create mode 100644 tests/nemo_text_processing/he/test_full_sentences.py create mode 100644 tests/nemo_text_processing/he/test_measure.py create mode 100644 tests/nemo_text_processing/he/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 tests/nemo_text_processing/he/test_time.py create mode 100644 tests/nemo_text_processing/he/test_whitelist.py create mode 100644 tests/nemo_text_processing/hi/__init__.py create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_address.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_address.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_electronic.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_serial.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_telephone.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/hi/data_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/hi/test_address.py create mode 100644 tests/nemo_text_processing/hi/test_cardinal.py create mode 100644 tests/nemo_text_processing/hi/test_date.py create mode 100644 tests/nemo_text_processing/hi/test_decimal.py create mode 100644 tests/nemo_text_processing/hi/test_electronic.py create mode 100644 tests/nemo_text_processing/hi/test_fraction.py create mode 100644 tests/nemo_text_processing/hi/test_measure.py create mode 100644 tests/nemo_text_processing/hi/test_money.py create mode 100644 tests/nemo_text_processing/hi/test_ordinal.py create mode 100644 tests/nemo_text_processing/hi/test_roman.py rename tests/nemo_text_processing/{zh/test_preprocess.py => hi/test_serial.py} (61%) create mode 100644 tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/hi/test_telephone.py create mode 100644 tests/nemo_text_processing/hi/test_time.py create mode 100644 tests/nemo_text_processing/hi/test_whitelist.py create mode 100644 tests/nemo_text_processing/hi/test_word.py create mode 100644 tests/nemo_text_processing/hi_en/__init__.py create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_address.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date_cased.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_electronic.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_telephone.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/hi_en/test_address.py create mode 100644 tests/nemo_text_processing/hi_en/test_cardinal.py create mode 100644 tests/nemo_text_processing/hi_en/test_date.py create mode 100644 tests/nemo_text_processing/hi_en/test_decimal.py create mode 100644 tests/nemo_text_processing/hi_en/test_electronic.py create mode 100644 tests/nemo_text_processing/hi_en/test_fraction.py create mode 100644 tests/nemo_text_processing/hi_en/test_measure.py create mode 100644 tests/nemo_text_processing/hi_en/test_money.py create mode 100644 tests/nemo_text_processing/hi_en/test_ordinal.py create mode 100644 tests/nemo_text_processing/hi_en/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 tests/nemo_text_processing/hi_en/test_telephone.py create mode 100644 tests/nemo_text_processing/hi_en/test_time.py create mode 100644 tests/nemo_text_processing/hi_en/test_whitelist.py create mode 100644 tests/nemo_text_processing/hi_en/test_word.py create mode 100644 tests/nemo_text_processing/hy/__init__.py create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/hy/data_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/hy/test_cardinal.py create mode 100644 tests/nemo_text_processing/hy/test_decimal.py create mode 100644 tests/nemo_text_processing/hy/test_fraction.py create mode 100644 tests/nemo_text_processing/hy/test_measure.py create mode 100644 tests/nemo_text_processing/hy/test_money.py create mode 100644 tests/nemo_text_processing/hy/test_ordinal.py create mode 100755 tests/nemo_text_processing/hy/test_sparrowhawk_inverse_text_normalization.sh create mode 100755 tests/nemo_text_processing/hy/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/hy/test_time.py create mode 100644 tests/nemo_text_processing/hy/test_whitelist.py create mode 100644 tests/nemo_text_processing/hy/test_word.py create mode 100644 tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt rename tests/nemo_text_processing/{zh/test_math.py => it/test_time.py} (62%) create mode 100644 tests/nemo_text_processing/ja/__init__.py create mode 100644 tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/ja/test_cardinal.py create mode 100644 tests/nemo_text_processing/ja/test_date.py create mode 100644 tests/nemo_text_processing/ja/test_decimal.py create mode 100644 tests/nemo_text_processing/ja/test_fraction.py create mode 100644 tests/nemo_text_processing/ja/test_ordinal.py create mode 100644 tests/nemo_text_processing/ja/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/ja/test_time.py create mode 100644 tests/nemo_text_processing/ko/__init__.py create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_telephone.txt create mode 100644 tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_electronic.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/ko/test_cardinal.py create mode 100644 tests/nemo_text_processing/ko/test_date.py create mode 100644 tests/nemo_text_processing/ko/test_decimal.py create mode 100644 tests/nemo_text_processing/ko/test_electronic.py create mode 100644 tests/nemo_text_processing/ko/test_fraction.py create mode 100644 tests/nemo_text_processing/ko/test_measure.py create mode 100644 tests/nemo_text_processing/ko/test_money.py create mode 100644 tests/nemo_text_processing/ko/test_ordinal.py create mode 100644 tests/nemo_text_processing/ko/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/ko/test_telephone.py create mode 100644 tests/nemo_text_processing/ko/test_time.py create mode 100644 tests/nemo_text_processing/mr/__init__.py create mode 100644 tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/mr/test_cardinal.py create mode 100644 tests/nemo_text_processing/mr/test_date.py create mode 100644 tests/nemo_text_processing/mr/test_decimal.py create mode 100644 tests/nemo_text_processing/mr/test_sparrowhawk_inverse_text_normalization.sh create mode 100644 tests/nemo_text_processing/mr/test_time.py create mode 100644 tests/nemo_text_processing/mr/test_word.py create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_electronic.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_telephone.txt create mode 100644 tests/nemo_text_processing/pt/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/pt/test_fraction.py create mode 100755 tests/nemo_text_processing/pt/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/rw/__init__.py create mode 100644 tests/nemo_text_processing/rw/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/rw/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/rw/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/rw/data_text_normalization/test_cases_word.txt create mode 100644 tests/nemo_text_processing/rw/test_cardinal.py create mode 100644 tests/nemo_text_processing/rw/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/rw/test_time.py create mode 100644 tests/nemo_text_processing/rw/test_whitelist.py create mode 100644 tests/nemo_text_processing/rw/test_word.py create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_abbreviation.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/se/data_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/se/test_abbreviation.py create mode 100644 tests/nemo_text_processing/se/test_measure.py create mode 100644 tests/nemo_text_processing/se/test_money.py create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_date.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_decimal.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_fraction.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_measure.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_money.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_ordinal.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_range.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_roman.txt create mode 100644 tests/nemo_text_processing/vi/data_text_normalization/test_cases_time.txt create mode 100644 tests/nemo_text_processing/vi/test_range.py create mode 100644 tests/nemo_text_processing/vi/test_roman.py create mode 100644 tests/nemo_text_processing/vi/test_sparrowhawk_normalization.sh create mode 100644 tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_word.txt delete mode 100644 tests/nemo_text_processing/zh/data_text_normalization/test_cases_math.txt delete mode 100644 tests/nemo_text_processing/zh/data_text_normalization/test_cases_preprocess.txt create mode 100644 tests/nemo_text_processing/zh/data_text_normalization/test_cases_whitelist.txt create mode 100644 tests/nemo_text_processing/zh/test_whitelist.py create mode 100644 tools/text_processing_deployment/sh_test.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bddd9abc3..fca523e58 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,25 +22,32 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v6.0.0 hooks: - id: check-yaml - id: check-case-conflict - id: detect-private-key - id: requirements-txt-fixer + - repo: https://github.com/PyCQA/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + args: + - --select=W605 + - repo: https://github.com/PyCQA/isort - rev: 4.3.21 + rev: 6.1.0 hooks: - id: isort name: Format imports args: [ --multi-line=3, --trailing-comma, --force-grid-wrap=0, --use-parentheses, --line-width=119, -rc, -ws ] exclude: docs/ - - repo: https://github.com/psf/black - rev: 19.10b0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.9.0 hooks: - id: black name: Format code args: [--skip-string-normalization, --line-length=119] - additional_dependencies: ['click==8.0.2'] + additional_dependencies: ['click>=8.0.2'] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..5bd44c418 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,51 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [r1.1.0] - 2024-08-20 + +### Added + +- DE TN Electronic recognizes social media handles `@Nvidia` (#177) +- Japanese ITN Cardinal, Date, Decimal, Fraction, Ordinal, Time and Whitelist coverage (#141) + +### Fixed + +- Fixed Fr TN Electronic processing issue from (#166) (#181) +- Fixed En TN Electronic processing issue from (#166) (#185) (#206) (#207) +- Fixed It TN Electronic processing issue from (#166) (#183) +- Fixed Hu TN Electronic processing issue from (#166) (#184) + + +### Changed + +- De TN Time coverage allows full stop delineation (e.g. `2.10h`) (#177) +- Es-En TN weights and data updated for unified (PnC) asr models (#143) +- En TN Expands coverage for technical terms for TTS processing (#167) +- En TN Money supports 'per unit' demarcation (e.g. `$20 per anum`) (#213) + + +## [r1.0.2] - 2024-05-03 + +### Added + +- Sentence level ZH (Mandarin Chinese) TN (#112) +- Enabled post-processing support for Sparrowhawk TN test (#147) + +### Fixed + +- `normalize_with_audio` text-field variable changed (#153) +- `run_evaluate` script for ITN updated for additional languages and casing (#164) + +### Changed + +- Docstring update (#157) + + +### Removed + +- Removed unused function from AR (Arabic) TN decimals (#165) + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5927d48b2..b62a5b7da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributions are welcome! -We do all of NeMo-text-processing's development in the open. Contributions from the open-source community are welcome. +We do all of NeMo-Text-Processing's development in the open. Contributions from the open-source community are welcome. # Pull Requests (PR) Guidelines @@ -22,4 +22,35 @@ We do all of NeMo-text-processing's development in the open. Contributions from 11) Optional: if you added a new language or a new feature please update the [NeMo documentation](https://github.com/NVIDIA/NeMo/blob/main/docs/source/nlp/text_normalization/wfst/wfst_text_normalization.rst) (lives in different repo). 12) Send your PR and request a review +# Notes for Language Contribution +1) `en/graph_utils.py` and `en/utils.py` are the de facto parents for all other `graph_util` and `utils` functions, respectively. Please refrain from duplicating code logics into new `graph_utils.py` and `utils.py` files and default to imports from `en/graph_utils.py` and `en/utils.py` instead. `LANG/graph_utils.py` and `LANG/utils.py` files should only contain new methods and variables. Not all new languages will require a submodule specific `graph_utils.py` or `utils.py` file. +2) NeMo-Text-Processing allows creation of FST graphs through two backends: the Python based library itself (via [Pynini](https://www.opengrm.org/twiki/bin/view/GRM/Pynini) backend) and C++ based [Sparrowhawk](https://github.com/google/sparrowhawk/tree/master) in an upstream repo. Due to the typical tradeoffs between these Python and C++ development [languages](https://www.youtube.com/watch?v=VioxsWYzoJk), the NeMo-Text-Processing library assumes development to be performed with the Python library for final deployment in Sparrowhawk/C++. This dual framework approach can lead to issues in development, notably in the case of tagging additional properties during tokenization. + +When writing taggers for semiotic classes, you may need to tag additional token properties (e.g. grammatical gender, case) for accurate verbalization. For example, the Spanish ordinal `21.º` carries masculine gender and is verbalized with a specific spelling. As such, it would be desired for the TN tagger to tokenize the string with the gender property included. + +Naively, one may be tempted to simply include the property string `gender: "masc"` and check for this string during the verbalization phase. **This is not advised.** While the NeMo-Text-Processing library itself will permit any custom string in the tagger, Sparrowhawk limits permissible strings, and will fail with custom property strings. Given the performance loss in not providing Sparrowhawk support, we cannot integrate new graphs that cause Sparrowhawk failure. As such, tagged properties should be limited to Sparrowhawk supported strings. + +For all classes, Sparrowhawk supports the `morphosyntactic_features` property, and it is recommended to default to this property for tagging additional features. For example: + +`21.º" -> ordinal { integer: "vigésimo primero" morphosyntactic_features: "masc" }` + +For additional Sparrowhawk supported properties by class, see [here](https://github.com/yzhang123/sparrowhawk/blob/test/src/proto/semiotic_classes.proto) + +N.B. The same limitation applies for novel semiotic classes as well. Only predefined classes are supported in Sparrowhawk. + +3) Between the tagging and verbalizing stages, both the NeMo-Text-Processing and Sparrowhawk engines permute order of tagged properties. That is, assuming the tagger parsed `1ᵉʳ juillet` as: + +`date { month: "juillet" day: "1" } }` + +the verbalizer will receive as input both + +`date { month: "juillet" day: "1" }` + +and + +`date { day: "1" month: "juillet" }` + +While this eases construction of verbalization graphs, permutation can be computationally expensive. If you know that the tagger output will not require permutation of token properties, you can improve model performance by including the `preserve_order: "true"` property: + +`date { day: "1" month: "juillet" preserve_order: "true" }` diff --git a/Jenkinsfile b/Jenkinsfile index 7c5663b74..a0dd7324a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,8 @@ pipeline { agent { docker { - image 'nvcr.io/nvidia/pytorch:22.12-py3' - args '--user 0:128 -v /home/jenkinsci:/home/jenkinsci -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""' + image 'tnitn_ci_py310:24.07' + args '-v /mnt/jenkins/jenkinsci/TestData:/home/jenkins/TestData -v $HOME/.cache:/root/.cache --shm-size=4g --entrypoint=""' } } options { @@ -10,31 +10,31 @@ pipeline { disableConcurrentBuilds(abortPrevious: true) } environment { - - AR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - DE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-14-23-0' - ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-13-23-1' - FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-16-23-1' - HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - SE_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-12-23-0' - DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0' - + AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0' + EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1' + ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' + ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' + HI_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-4' + FR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-07-25-0' + HU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-16-24-0' + PT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/05-01-26-1' + RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' + VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-29-25-0' + SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' + SE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' + ZH_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-13-24-0' + IT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-22-24-0' + HE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-24-25-0' + HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' + MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' + JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' + KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' + KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-29-26-1' + DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { - stage('Add git safe directory'){ - steps{ - sh 'git config --global --add safe.directory /var/lib/jenkins/workspace/NTP_$GIT_BRANCH' - sh 'git config --global --add safe.directory /home/jenkinsci/workspace/NTP_$GIT_BRANCH' - } - } stage('PyTorch version') { steps { @@ -43,14 +43,6 @@ pipeline { } } - stage('Install test requirements') { - steps { - sh 'apt-get update && apt-get install -y bc' - } - } - - - stage('NeMo Installation') { steps { sh './reinstall.sh release' @@ -62,7 +54,10 @@ pipeline { when { anyOf { branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' + } } failFast true @@ -90,11 +85,41 @@ pipeline { } } + stage('L0: Create HI TN/ITN Grammars') { + when { + anyOf { + branch 'main' + branch 'staging/**' + branch 'staging_*' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L0: Hi TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hi --text="१" --cache_dir ${HI_TN_CACHE}' + } + } + stage('L0: Hi ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hi --text="एक" --cache_dir ${HI_TN_CACHE}' + } + } + stage('L0: Codeswitched HI/EN ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hi_en --text="एक" --cache_dir ${HI_EN_TN_CACHE}' + } + } + } + } - stage('L0: Create DE TN/ITN Grammars') { + stage('L0: Create DE/ES TN/ITN Grammars') { when { anyOf { branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -102,27 +127,14 @@ pipeline { parallel { stage('L0: DE TN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=de --text="1" --cache_dir ${DEFAULT_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=de --text="1" --cache_dir ${DE_TN_CACHE}' } } stage('L0: DE ITN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=de --text="ein hundert " --cache_dir ${DEFAULT_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=de --text="ein hundert " --cache_dir ${DE_TN_CACHE}' } } - - } - } - - stage('L0: Create ES TN/ITN Grammars') { - when { - anyOf { - branch 'main' - changeRequest target: 'main' - } - } - failFast true - parallel { stage('L0: ES TN grammars') { steps { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=es --text="1" --cache_dir ${ES_TN_CACHE}' @@ -133,33 +145,20 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=es --text="ciento uno " --cache_dir ${ES_TN_CACHE}' } } - - } - } - - stage('L0: Create Codeswitched ES/EN TN/ITN Grammars') { - when { - anyOf { - branch 'main' - changeRequest target: 'main' - } - } - failFast true - parallel { - - stage('L0: ES/EN ITN grammars') { + stage('L0: Codeswitched ES/EN ITN grammars') { steps { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=es_en --text="ciento uno " --cache_dir ${ES_EN_TN_CACHE}' } } - } } stage('L0: Create AR TN/ITN Grammars') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -175,115 +174,161 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ar --text="اثنان " --cache_dir ${AR_TN_CACHE}' } } - } } - stage('L0: Create FR TN/ITN Grammars') { + stage('L0: Create FR TN/ITN & VI TN/ITN & HU TN & IT TN') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } failFast true parallel { - // stage('L0: FR TN grammars') { - // steps { - // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=fr --text="2" --cache_dir ${FR_TN_CACHE}' - // } - // } + stage('L0: FR TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=fr --text="2" --cache_dir ${FR_TN_CACHE}' + } + } stage('L0: FR ITN grammars') { steps { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=fr --text="cent " --cache_dir ${FR_TN_CACHE}' } } - + stage('L0: VI ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=vi --text="một ngàn " --cache_dir ${VI_TN_CACHE}' + } + } + stage('L0: VI TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=vi --text="100" --cache_dir ${VI_TN_CACHE}' + } + } + stage('L0: HU TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hu --text="100" --cache_dir ${HU_TN_CACHE}' + } + } + stage('L0: IT TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=it --text="122" --cache_dir ${IT_TN_CACHE}' + } + } } } - stage('L0: Create HU TN/ITN Grammars') { + + stage('L0: Create RU TN/ITN Grammars & SV & PT') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } failFast true parallel { - stage('L0: HU TN grammars') { + stage('L0: RU TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --lang=ru --text="03" --cache_dir ${RU_TN_CACHE}' + } + } + stage('L0: RU ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ru --text="три " --cache_dir ${RU_TN_CACHE}' + } + } + stage('L0: SV TN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hu --text="100" --cache_dir ${HU_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=sv --text="100" --cache_dir ${SV_TN_CACHE}' + } + } + // stage('L0: SV ITN grammars') { + // steps { + // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=sv --text="hundra " --cache_dir ${SV_TN_CACHE}' + // } + // } + stage('L0: PT TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=pt --text="2" --cache_dir ${PT_TN_CACHE}' + } + } + stage('L0: PT ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=pt --text="dez " --cache_dir ${PT_TN_CACHE}' } } - // stage('L0: HU ITN grammars') { - // steps { - // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hu --text="száz " --cache_dir ${HU_TN_CACHE}' - // } - // } } } - stage('L0: Create VI TN/ITN Grammars') { + stage('L0: Create HE ITN Grammar') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } failFast true parallel { - // stage('L0: VI TN grammars') { - // steps { - // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=vi --text="2" --cache_dir ${VI_TN_CACHE}' - // } - // } - stage('L0: VI ITN grammars') { + stage('L0: HE ITN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=vi --text="một ngàn " --cache_dir ${VI_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=he --text="ת " --cache_dir ${HE_TN_CACHE}' } } - } } - - stage('L0: Create PT TN/ITN Grammars') { + stage('L0: Create HY TN/ITN Grammars & MR') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } failFast true parallel { - // stage('L0: PT TN grammars') { - // steps { - // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=pt --text="2" --cache_dir ${DEFAULT_TN_CACHE}' - // } - // } - stage('L0: PT ITN grammars') { + stage('L0: MR ITN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=pt --text="dez " --cache_dir ${PT_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=mr --text="शून्य " --cache_dir ${MR_TN_CACHE}' + } + } + stage('L0: HY TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hy --text="6" --cache_dir ${HY_TN_CACHE}' + } + } + stage('L0: HY ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hy --text="վեց " --cache_dir ${HY_TN_CACHE}' } } - } } - stage('L0: Create RU TN/ITN Grammars') { + stage('L0: Create ZH TN/ITN Grammar') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } failFast true parallel { - stage('L0: RU TN grammars') { + stage('L0: ZH ITN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize_with_audio.py --lang=ru --text="03" --cache_dir ${RU_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=zh --text="你" --cache_dir ${ZH_TN_CACHE}' } } - stage('L0: RU ITN grammars') { + stage('L0: ZH TN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ru --text="три " --cache_dir ${RU_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=zh --text="6" --cache_dir ${ZH_TN_CACHE}' } } } @@ -299,64 +344,62 @@ pipeline { parallel { stage('L0: SE TN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=se --text="100" --cache_dir ${SV_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=se --text="100" --cache_dir ${SE_TN_CACHE}' } } - // stage('L0: SE ITN grammars') { - // steps { - // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=se --text="hundra " --cache_dir ${SV_TN_CACHE}' - // } - // } } } - stage('L0: Create SV TN/ITN Grammars') { + stage('L0: Create JA ITN Grammars') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } failFast true parallel { - stage('L0: SV TN grammars') { - steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=sv --text="100" --cache_dir ${SV_TN_CACHE}' + stage('L0: JA ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ja --text="100" --cache_dir ${JA_TN_CACHE}' } } - // stage('L0: SV ITN grammars') { - // steps { - // sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=sv --text="hundra " --cache_dir ${SV_TN_CACHE}' - // } - // } } } - stage('L0: Create ZH TN/ITN Grammars') { + stage('L0: Create KO TN/ITN Grammars') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } failFast true parallel { - stage('L0: ZH TN grammars') { - steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=zh --text="你" --cache_dir ${ZH_TN_CACHE}' + stage('L0: KO ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ko --text="백" --cache_dir ${KO_TN_CACHE}' } } - stage('L0: ZH ITN grammars') { + stage('L0: KO TN grammars') { steps { - sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=zh --text="二零零二年一月二十八日 " --cache_dir ${ZH_TN_CACHE}' + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=ko --text="100" --cache_dir ${KO_TN_CACHE}' } } } } + // L1 Tests starts here + stage('L1: TN/ITN Tests CPU') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -377,11 +420,21 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_TN_CACHE}' } } + stage('L1: Run all HI TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hi/ -m "not pleasefixme" --cpu --tn_cache_dir ${HI_TN_CACHE}' + } + } stage('L1: Run all Codeswitched ES/EN TN/ITN tests (restore grammars from cache)') { steps { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es_en/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_EN_TN_CACHE}' } } + stage('L1: Run all Codeswitched HI/EN TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hi_en/ -m "not pleasefixme" --cpu --tn_cache_dir ${HI_EN_TN_CACHE}' + } + } stage('L1: Run all AR TN/ITN tests (restore grammars from cache)') { steps { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ar/ -m "not pleasefixme" --cpu --tn_cache_dir ${AR_TN_CACHE}' @@ -417,13 +470,75 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/zh/ -m "not pleasefixme" --cpu --tn_cache_dir ${ZH_TN_CACHE}' } } + stage('L1: Run all JA TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ja/ -m "not pleasefixme" --cpu --tn_cache_dir ${JA_TN_CACHE}' + } + } + stage('L1: Run all MR ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/mr/ -m "not pleasefixme" --cpu --tn_cache_dir ${MR_TN_CACHE}' + } + } + stage('L1: Run all HY TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hy/ -m "not pleasefixme" --cpu --tn_cache_dir ${HY_TN_CACHE}' + } + } + stage('L1: Run all HE TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/he/ -m "not pleasefixme" --cpu --tn_cache_dir ${HE_TN_CACHE}' + } + } + stage('L1: Run all KO TN/ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ko/ -m "not pleasefixme" --cpu --tn_cache_dir ${KO_TN_CACHE}' + } + } } } + stage('L2: EN Sparrowhawk Tests') { + when { + anyOf { + branch 'main' + branch 'staging/**' + branch 'staging_*' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L2: EN ITN Run Sparrowhawk test - Lower Cased Input') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_lower_cased `pwd`' + + } + } + stage('L2: EN ITN Run Sparrowhawk test - Cased Input') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_itn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_ITN_cased --LANGUAGE="en"' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_inverse_text_normalization_cased.sh /workspace/sparrowhawk/documentation/grammars_en_itn_grammars_cased `pwd`' + + } + } + stage('L2: EN TN Run Sparrowhawk test') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${EN_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="en" ' + sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/en && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars_en_tn_grammars_cased `pwd`' + } + } + + } + } + stage('L2: NeMo text processing') { when { anyOf { - branch 'main' + branch 'main' + branch 'staging/**' + branch 'staging_*' changeRequest target: 'main' } } @@ -431,23 +546,23 @@ pipeline { parallel { stage('L2: Eng TN') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && \ cd tools/text_processing_deployment && python pynini_export.py --output=$NORM_OUTPUT_DIR --grammars=tn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $NORM_OUTPUT_DIR && echo ".far files created "|| exit 1' - sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ - cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkinsci/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ + cd nemo_text_processing/text_normalization/ && python normalize.py --input_file=/home/jenkins/TestData/text_norm/ci/test.txt --input_case="lower_cased" --language=en --output_file=$NORM_OUTPUT_DIR/test.pynini.txt --verbose && \ cat $NORM_OUTPUT_DIR/test.pynini.txt && \ - cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \ + cmp --silent $NORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_norm/ci/test_goal_py.txt || exit 1 && \ rm -rf $NORM_OUTPUT_DIR' } } stage('L2: Eng ITN export') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && \ cd tools/text_processing_deployment && python pynini_export.py --output=$DENORM_OUTPUT_DIR --grammars=itn_grammars --cache_dir ${EN_TN_CACHE} --language=en && ls -R $DENORM_OUTPUT_DIR && echo ".far files created "|| exit 1' - sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ - cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkinsci/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \ - cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkinsci/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ + cd nemo_text_processing/inverse_text_normalization/ && python inverse_normalize.py --input_file=/home/jenkins/TestData/text_denorm/ci/test.txt --language=en --output_file=$DENORM_OUTPUT_DIR/test.pynini.txt --verbose && \ + cmp --silent $DENORM_OUTPUT_DIR/test.pynini.txt /home/jenkins/TestData/text_denorm/ci/test_goal_py.txt || exit 1 && \ rm -rf $DENORM_OUTPUT_DIR' } } @@ -455,18 +570,18 @@ pipeline { stage('L2: Eng alignment TN') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && NORM_OUTPUT_DIR=/home/jenkins/TestData/text_norm/output_${TIME} && mkdir $NORM_OUTPUT_DIR && \ cd nemo_text_processing/fst_alignment && python alignment.py --text="2615 Forest Av, 90501 CA, Santa Clara. 10kg, 12/16/2018" --grammar=tn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_tn_True_deterministic_cased__tokenize.far 2>&1 | tee $NORM_OUTPUT_DIR/pred.txt && \ - cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \ + cmp --silent $NORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_norm/ci/alignment_gold.txt || exit 1 && \ rm -rf $NORM_OUTPUT_DIR' } } stage('L2: Eng alignment ITN') { steps { - sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkinsci/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ + sh 'TIME=`date +"%Y-%m-%d-%T"` && DENORM_OUTPUT_DIR=/home/jenkins/TestData/text_denorm/output_${TIME} && mkdir $DENORM_OUTPUT_DIR && \ cd nemo_text_processing/fst_alignment && python alignment.py --text="one million twenty three thousand two hundred eleven ten kilograms one hundred twenty three dollars and twenty five cents" --grammar=itn --rule=tokenize_and_classify --fst=${EN_TN_CACHE}/en_itn_lower_cased.far 2>&1 | tee $DENORM_OUTPUT_DIR/pred.txt && \ - cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkinsci/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \ + cmp --silent $DENORM_OUTPUT_DIR/pred.txt /home/jenkins/TestData/text_denorm/ci/alignment_gold.txt || exit 1 && \ rm -rf $DENORM_OUTPUT_DIR' } } diff --git a/README.md b/README.md index 03020d557..8bbb5c617 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Installation We recommend setting up a fresh Conda environment to install NeMo-text-processing. ```bash -conda create --name nemo_tn python==3.8 +conda create --name nemo_tn python==3.10 conda activate nemo_tn ``` @@ -54,7 +54,7 @@ pip install nemo_text_processing **_NOTE:_** This should work on any Linux OS with x86_64. Pip installation on MacOS and Windows are not supported due to the dependency [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini). On a platform other than Linux x86_64, installing from Pip tries to compile Pynini from scratch, and requires OpenFst headers and libraries to be in the expected place. So if it's working for you, it's because you happen to have installed OpenFst in the right way in the right place. So if you want to Pip install Pynini on MacOS, you have to have pre-compiled and pre-installed OpenFst. The Pynini README for that version should tell you which version it needs and what `--enable-foo` flags to use. Instead, we recommend you to use conda-forge to install Pynini on MacOS or Windows: -`conda install -c conda-forge pynini=2.1.5`. +`conda install -c conda-forge pynini=2.1.6.post1`. ### Pip from source diff --git a/nemo_text_processing/fst_alignment/alignment.py b/nemo_text_processing/fst_alignment/alignment.py index 007d6d9c9..c16aec079 100644 --- a/nemo_text_processing/fst_alignment/alignment.py +++ b/nemo_text_processing/fst_alignment/alignment.py @@ -21,7 +21,6 @@ import pynini from pynini import Far - """ This files takes 1. Far file containing a fst graph created by TN or ITN 2. entire string. Optionally: 3. start position of substring 4. end (exclusive) position of substring @@ -97,10 +96,11 @@ def parse_args(): EPS = "" -WHITE_SPACE = "\u23B5" +WHITE_SPACE = "\u23b5" ITN_MODE = "itn" TN_MODE = "tn" -tn_itn_symbols = list(string.ascii_letters + string.digits) + list("$\:+-=") +tn_item_special_chars = ["$", "\\", ":", "+", "-", "="] +tn_itn_symbols = list(string.ascii_letters + string.digits) + tn_item_special_chars def get_word_segments(text: str) -> List[List[int]]: @@ -200,7 +200,7 @@ def indexed_map_to_output(alignment: List[tuple], start: int, end: int, mode: st alignment: alignment generated by FST with shortestpath, is longer than original string since including eps transitions start: inclusive start position in input string end: exclusive end position in input string - mode: grammar type for either tn or itn + mode: grammar type for either tn or itn Returns: output_og_start_index: inclusive start position in output string diff --git a/nemo_text_processing/hybrid/mlm_scorer.py b/nemo_text_processing/hybrid/mlm_scorer.py index 2986f3562..b2c94598e 100644 --- a/nemo_text_processing/hybrid/mlm_scorer.py +++ b/nemo_text_processing/hybrid/mlm_scorer.py @@ -93,7 +93,7 @@ def score_sentence(self, sentence: str): def __mask_text__(self, idx: int, tokens: List[str]): """ - replaces string at index idx in list `tokens` with a masked token and returns the modified list. + replaces string at index idx in list `tokens` with a masked token and returns the modified list. """ masked = tokens.copy() masked[idx] = self.MASK_LABEL diff --git a/nemo_text_processing/hybrid/model_utils.py b/nemo_text_processing/hybrid/model_utils.py index f40b56ba0..b81d59b2a 100644 --- a/nemo_text_processing/hybrid/model_utils.py +++ b/nemo_text_processing/hybrid/model_utils.py @@ -17,9 +17,10 @@ import re from typing import List, Union -from nemo_text_processing.hybrid.mlm_scorer import MLMScorer from tqdm import tqdm +from nemo_text_processing.hybrid.mlm_scorer import MLMScorer + try: import torch except ImportError as e: @@ -56,15 +57,15 @@ def get_masked_score(text, model, do_lower=True): If multiple tokens are present, multiple variants of the text are created where all but one ambiguous semiotic tokens are masked to avoid unwanted reinforcement of neighboring semiotic tokens.""" text = text.lower() if do_lower else text - spans = re.findall("<\s.+?\s>", text) + spans = re.findall(r"<\s.+?\s>", text) if len(spans) > 0: text_with_mask = [] - for match in re.finditer("<\s.+?\s>", text): + for match in re.finditer(r"<\s.+?\s>", text): new_text = ( text[: match.span()[0]] + match.group().replace("< ", "").replace(" >", "") + text[match.span()[1] :] ) - new_text = re.sub("<\s.+?\s>", model.MASK_LABEL, new_text) + new_text = re.sub(r"<\s.+?\s>", model.MASK_LABEL, new_text) text_with_mask.append(new_text) text = text_with_mask @@ -73,11 +74,11 @@ def get_masked_score(text, model, do_lower=True): def _get_ambiguous_positions(sentences: List[str]): """returns None or index list of ambigous semiotic tokens for list of sentences. - E.g. if sentences = ["< street > < three > A", "< saint > < three > A"], it returns [1, 0] since only + E.g. if sentences = ["< street > < three > A", "< saint > < three > A"], it returns [1, 0] since only the first semiotic span / is ambiguous.""" - l_sets = [set([x]) for x in re.findall("<\s.+?\s>", sentences[0])] + l_sets = [set([x]) for x in re.findall(r"<\s.+?\s>", sentences[0])] for sentence in sentences[1:]: - spans = re.findall("<\s.+?\s>", sentence) + spans = re.findall(r"<\s.+?\s>", sentence) if len(spans) != len(l_sets): return None for i in range(len(spans)): @@ -114,7 +115,7 @@ def score_options(sentences: List[str], context_len, model, do_lower=True): scores.append(av_score) elif isinstance(sent, str): # in case of full context if ambiguous_positions: - matches = list(re.finditer("<\s.+?\s>", sent)) + matches = list(re.finditer(r"<\s.+?\s>", sent)) for match, pos in zip(matches[::-1], ambiguous_positions[::-1]): if not pos: sent = ( diff --git a/nemo_text_processing/hybrid/utils.py b/nemo_text_processing/hybrid/utils.py index 699a5b647..39a3fc2c6 100644 --- a/nemo_text_processing/hybrid/utils.py +++ b/nemo_text_processing/hybrid/utils.py @@ -23,11 +23,12 @@ import pandas as pd import pynini -from nemo_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from pynini.lib.rewrite import top_rewrite from tqdm import tqdm +from nemo_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + DELIMITER = '~~' cardinal_graph = CardinalFst(input_case="cased").graph_no_exception @@ -389,8 +390,8 @@ def clean_post_norm( def clean_libri_tts(target: str): """ - Replace abbreviations in LibriTTS dataset - """ + Replace abbreviations in LibriTTS dataset + """ # Normalized text in LibriTTS by Google which contains abbreviations from `libri_sometimes_converts_abbrs` sometimes wasn't converted. libri_sometimes_converts_abbrs = {"St.": "saint", "Rev.": "reverend"} @@ -481,7 +482,7 @@ def remove_punctuation(text: str, remove_spaces=True, do_lower=True, lang="en", text = re.sub(r" +", " ", text) if remove_spaces: - text = text.replace(" ", "").replace("\u00A0", "").strip() + text = text.replace(" ", "").replace("\u00a0", "").strip() if do_lower: text = text.lower() @@ -514,7 +515,11 @@ def _relax_diff(text): return acceptable -def get_labels(targets: List[str], norm_texts_weights: List[Tuple[str, str]], lang="en",) -> List[List[str]]: +def get_labels( + targets: List[str], + norm_texts_weights: List[Tuple[str, str]], + lang="en", +) -> List[List[str]]: """ Assign labels to generated normalization options (1 - for ground truth, 0 - other options) Args: @@ -604,7 +609,14 @@ def print_df(df): prints data frame """ with pd.option_context( - "display.max_rows", None, "display.max_columns", None, "display.width", 1000, "display.max_colwidth", 400, + "display.max_rows", + None, + "display.max_columns", + None, + "display.width", + 1000, + "display.max_colwidth", + 400, ): print(df) @@ -640,7 +652,7 @@ def get_diff(a: str, b: str): def diff_pred_gt(pred: str, gt: str): """returns list of different substrings between prediction and gt - relies on that prediction uses '< ' ' >' + relies on that prediction uses '< ' ' >' Args: pred (str): prediction @@ -648,7 +660,7 @@ def diff_pred_gt(pred: str, gt: str): Returns: list of Tuple(pred start and end, gt start and end) subsections - + e.g. pred="< Edward third >., king Our own . loss had been < two thousand two hundred >" gt ="Edward III., king Our own loss had been twenty two hundred" --> [([0, 16], [0, 10]), ([32, 34], [26, 26]), ([48, 76], [40, 58])] diff --git a/nemo_text_processing/hybrid/wfst_lm_rescoring.py b/nemo_text_processing/hybrid/wfst_lm_rescoring.py index e3c0bd02d..7f001e6a2 100644 --- a/nemo_text_processing/hybrid/wfst_lm_rescoring.py +++ b/nemo_text_processing/hybrid/wfst_lm_rescoring.py @@ -25,9 +25,10 @@ import pandas as pd import utils from joblib import Parallel, delayed -from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio from tqdm import tqdm +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + parser = argparse.ArgumentParser(description="Re-scoring") parser.add_argument("--lang", default="en", type=str, choices=["en"]) parser.add_argument("--n_tagged", default=100, type=int, help="Number WFST options") @@ -72,7 +73,7 @@ def threshold_weights(norm_texts_weights, delta: float = 0.2): delta: delta to add to minimum weight in options to compose upper limit for threshhold returns: - filter list of same format as input + filter list of same format as input """ # threshold value is factor applied to lowest/first weight of all normalization options for every input res = [] diff --git a/nemo_text_processing/inverse_text_normalization/ar/__init__.py b/nemo_text_processing/inverse_text_normalization/ar/__init__.py index 53026c2da..e56a8f9af 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/ar/__init__.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/__init__.py index 53026c2da..e56a8f9af 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/__init__.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/cardinal.py index a9a6869bf..2c58df6a9 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/cardinal.py @@ -13,16 +13,17 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_SPACE, GraphFst + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals e.g. سالب تسعة وتسعون -> cardinal { integer: "99" negative: "-" } } - Numbers below thirteen are not converted. - Args: + Numbers below thirteen are not converted. + Args: tn_cardinal: cardinal FST for TN """ @@ -32,7 +33,9 @@ def __init__(self, tn_cardinal): self.graph = pynini.invert(tn_cardinal.cardinal_numbers).optimize() optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("سالب", '"-"') + NEMO_SPACE, 0, 1, + pynutil.insert("negative: ") + pynini.cross("سالب", '"-"') + NEMO_SPACE, + 0, + 1, ) final_graph = optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py index fa14c4629..3b22ece05 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_SPACE, GraphFst, delete_extra_space, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): @@ -35,7 +36,9 @@ def __init__(self, tn_decimal): super().__init__(name="decimal", kind="classify") optional_graph_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("سالب", '"true"') + delete_extra_space, 0, 1, + pynutil.insert("negative: ") + pynini.cross("سالب", '"true"') + delete_extra_space, + 0, + 1, ) graph_fractional_part = pynini.invert(tn_decimal.graph_fractional).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py index be18ac16f..db14cc9a8 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -21,14 +23,13 @@ insert_space, ) from nemo_text_processing.text_normalization.ar.utils import get_abs_path -from pynini.lib import pynutil class FractionFst(GraphFst): """ Finite state transducer for classifying fraction e.g. واحد و نصف -> tokens { integer_part: "1" numerator: "1" denominator: "2" } - + Args: tn_cardinal: TN cardinal tagger diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py index ac3fc7fbe..2f047240e 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst, convert_space, delete_extra_space from nemo_text_processing.text_normalization.ar.taggers.measure import unit_singular -from pynini.lib import pynutil class MeasureFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/money.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/money.py index e13b5a45a..50eaa5d6f 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -23,7 +25,6 @@ insert_space, ) from nemo_text_processing.text_normalization.ar.taggers.money import ar_cur, maj_singular, min_plural, min_singular -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/punctuation.py index 798546ccf..5fe5a8411 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/punctuation.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/punctuation.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst + class PunctuationFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py index 8f6dd0782..97818c7bb 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.ar.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.ar.taggers.decimal import DecimalFst from nemo_text_processing.inverse_text_normalization.ar.taggers.fraction import FractionFst @@ -31,7 +32,7 @@ ) from nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -62,9 +63,9 @@ def __init__( far_file = os.path.join(cache_dir, f"ar_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") tn_classify = TNClassifyFst( input_case='cased', deterministic=True, cache_dir=cache_dir, overwrite_cache=True ) @@ -109,4 +110,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/ar/taggers/word.py b/nemo_text_processing/inverse_text_normalization/ar/taggers/word.py index bf9095f6b..83ec8f76a 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/ar/taggers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ar/utils.py b/nemo_text_processing/inverse_text_normalization/ar/utils.py index ca6210150..67594bf55 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/utils.py +++ b/nemo_text_processing/inverse_text_normalization/ar/utils.py @@ -27,7 +27,7 @@ def num_to_word(x: Union[str, int]): Args x: integer - Returns: spoken representation + Returns: spoken representation """ if isinstance(x, int): x = str(x) @@ -41,7 +41,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/__init__.py index 1f7941250..8944d9e6c 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/__init__.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - import pynini + from nemo_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.utils.logging import logger diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/cardinal.py index 5ced7907d..2145bd935 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/cardinal.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/decimal.py index 0b5f5ed2c..842f6e3f5 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/fraction.py index a36533bee..3a7f885d5 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/fraction.py @@ -14,6 +14,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SPACE, @@ -21,7 +23,6 @@ delete_space, delete_zero_or_one_space, ) -from pynini.lib import pynutil class FractionFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py index 78299091b..f2eeee5ce 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/money.py index 084e740d4..51d5de412 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_CHAR, NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize_final.py index 17aa67826..7f557096b 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize_final.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.ar.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.ar.verbalizers.word import WordFst from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now """ diff --git a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/word.py index 11da24fab..76a02eefe 100644 --- a/nemo_text_processing/inverse_text_normalization/ar/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/ar/verbalizers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WordFst(GraphFst): """ @@ -26,6 +27,6 @@ def __init__(self): super().__init__(name="word", kind="verbalize") chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index e0b066173..46fdca4e3 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst + class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals. Numbers below ten are not converted. + Finite state transducer for classifying cardinals. Numbers below ten are not converted. Allows both compound numeral strings or separated by whitespace. "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. @@ -31,7 +32,7 @@ class CardinalFst(GraphFst): e.g. ein tausend -> cardinal { integer: "1000" } } e.g. eintausend -> cardinal { integer: "1000" } } e.g. ein tausend zwanzig -> cardinal { integer: "1020" } } - + Args: tn_cardinal_tagger: TN cardinal tagger """ diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/date.py b/nemo_text_processing/inverse_text_normalization/de/taggers/date.py index b65be41e4..916b9413d 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, @@ -20,7 +22,6 @@ GraphFst, convert_space, ) -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/decimal.py index e6d562d4b..53354e3ed 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.taggers.decimal import get_quantity, quantities from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst -from pynini.lib import pynutil class DecimalFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/de/taggers/electronic.py index 849d6c843..dc9f96bd1 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/electronic.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class ElectronicFst(GraphFst): """ Finite state transducer for classifying electronic: email addresses, etc. e.g. c d f eins at a b c punkt e d u -> tokens { name: "cdf1.abc.edu" } - + Args: tn_electronic_tagger: TN eletronic tagger tn_electronic_verbalizer: TN eletronic verbalizer diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/de/taggers/fraction.py index 6960a6663..960c9ffa9 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/fraction.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, convert_space, delete_space, ) -from pynini.lib import pynutil class FractionFst(GraphFst): @@ -28,7 +29,7 @@ class FractionFst(GraphFst): e.g. ein halb -> tokens { name: "1/2" } e.g. ein ein halb -> tokens { name: "1 1/2" } e.g. drei zwei ein hundertstel -> tokens { name: "3 2/100" } - + Args: itn_cardinal_tagger: ITN cardinal tagger tn_fraction_verbalizer: TN fraction verbalizer diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/de/taggers/measure.py index 86eff3ff2..dbfff31b4 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.taggers.measure import singular_to_plural, unit_singular from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_SIGMA, @@ -21,7 +23,6 @@ delete_extra_space, delete_space, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/money.py b/nemo_text_processing/inverse_text_normalization/de/taggers/money.py index dad06de1c..c53b865ed 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.taggers.money import maj_singular, min_plural, min_singular from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, @@ -23,7 +25,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/ordinal.py index 300dd9010..52b8ffa0b 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/de/taggers/telephone.py index ddc3391d4..dd7f79878 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/telephone.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space, insert_space + class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone numbers, e.g. + Finite state transducer for classifying telephone numbers, e.g. null vier eins eins eins zwei drei vier eins zwei drei vier -> tokens { name: "(0411) 1234-1234" } - + Args: tn_cardinal_tagger: TN Cardinal Tagger """ diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/time.py b/nemo_text_processing/inverse_text_normalization/de/taggers/time.py index c747202b9..db2edb66b 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/time.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst + class TimeFst(GraphFst): """ @@ -30,7 +31,7 @@ class TimeFst(GraphFst): e.g. drei vor zwölf -> time { minutes: "57" hours: "11" } e.g. drei nach zwölf -> time { minutes: "3" hours: "12" } e.g. drei uhr zehn minuten zehn sekunden -> time { hours: "3" hours: "10" sekunden: "10"} - + Args: tn_time_verbalizer: TN time verbalizer """ diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py index c1025db44..1d60d071a 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.de.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.de.taggers.date import DateFst from nemo_text_processing.inverse_text_normalization.de.taggers.decimal import DecimalFst @@ -46,7 +47,7 @@ delete_space, generator_main, ) -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -78,9 +79,9 @@ def __init__( far_file = os.path.join(cache_dir, f"de_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") tn_cardinal_tagger = TNCardinalTagger(deterministic=False) tn_date_tagger = TNDateTagger(cardinal=tn_cardinal_tagger, deterministic=False) tn_decimal_tagger = TNDecimalTagger(cardinal=tn_cardinal_tagger, deterministic=False) @@ -147,4 +148,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/de/taggers/whitelist.py index 18c6e0502..c99d79c30 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/whitelist.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space + class WhiteListFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index 92a83625a..b13382a8e 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/decimal.py index ff3839533..8c4ac951e 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/measure.py index 90f43e7e6..14afbbee5 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py index f0377d4fa..f8bbc0793 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/time.py index 13a013db5..ac67928ce 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/time.py @@ -13,16 +13,17 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, delete_space + class TimeFst(GraphFst): """ Finite state transducer for verbalizing time, e.g. time { hours: "8" minutes: "30" zone: "e s t" } -> 08:30 Uhr est time { hours: "8" } -> 8 Uhr - time { hours: "8" minutes: "30" seconds: "10" } -> 08:30:10 Uhr + time { hours: "8" minutes: "30" seconds: "10" } -> 08:30:10 Uhr """ def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py index 779a00a42..beb9b1e7c 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize_final.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.de.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "jetzt" } tokens { name: "ist" } tokens { time { hours: "12" minutes: "30" } } -> jetzt ist 12:30 Uhr """ diff --git a/nemo_text_processing/inverse_text_normalization/en/clean_eval_data.py b/nemo_text_processing/inverse_text_normalization/en/clean_eval_data.py index d9bc2fccb..e9dd16034 100644 --- a/nemo_text_processing/inverse_text_normalization/en/clean_eval_data.py +++ b/nemo_text_processing/inverse_text_normalization/en/clean_eval_data.py @@ -16,6 +16,7 @@ from typing import List import regex as re + from nemo_text_processing.text_normalization.data_loader_utils import ( EOS_TYPE, Instance, @@ -23,7 +24,6 @@ training_data_to_sentences, ) - """ This file is for evaluation purposes. filter_loaded_data() cleans data (list of instances) for inverse text normalization. Filters and cleaners can be specified for each semiotic class individually. @@ -67,7 +67,7 @@ def process(self, instance: Instance) -> Instance: Args: processes given instance with process function - + Returns: processed instance if instance belongs to expected class type or original instance """ if instance.token_type != self.class_type: diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/en/taggers/cardinal.py index 8eb98bd5d..5eea89af1 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/cardinal.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path, num_to_word from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -27,7 +29,6 @@ capitalized_input_graph, delete_space, ) -from pynini.lib import pynutil class CardinalFst(GraphFst): @@ -206,7 +207,10 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED): + graph_in_thousands ) - graph = pynini.union((graph_int | graph_ind) + delete_space + graph_hundreds, graph_zero,) + graph = pynini.union( + (graph_int | graph_ind) + delete_space + graph_hundreds, + graph_zero, + ) graph = graph @ pynini.union( pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), "0" @@ -242,7 +246,7 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED): self.fst = final_graph.optimize() def delete_word(self, word: str): - """ Capitalizes word for `cased` input""" + """Capitalizes word for `cased` input""" delete_graph = pynutil.delete(word).optimize() if self.input_case == INPUT_CASED: if len(word) > 0: diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/date.py b/nemo_text_processing/inverse_text_normalization/en/taggers/date.py index fadde9cbd..b1ace40ce 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/date.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -25,7 +27,6 @@ delete_extra_space, delete_space, ) -from pynini.lib import pynutil graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).optimize() graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize() @@ -136,7 +137,7 @@ def _get_thousands_graph(): class DateFst(GraphFst): """ - Finite state transducer for classifying date, + Finite state transducer for classifying date, e.g. january fifth twenty twelve -> date { month: "january" day: "5" year: "2012" preserve_order: true } e.g. the fifth of january twenty twelve -> date { day: "5" month: "january" year: "2012" preserve_order: true } e.g. twenty twenty -> date { year: "2012" preserve_order: true } @@ -164,7 +165,11 @@ def __init__(self, ordinal: GraphFst, input_case: str): + pynutil.add_weight(year_graph, -YEAR_WEIGHT) + pynutil.insert("\"") ) - optional_graph_year = pynini.closure(graph_year, 0, 1,) + optional_graph_year = pynini.closure( + graph_year, + 0, + 1, + ) graph_mdy = month_graph + ( (delete_extra_space + day_graph) | graph_year | (delete_extra_space + day_graph + graph_year) ) diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/en/taggers/decimal.py index f6a730ad6..6e5de2418 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/decimal.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -29,7 +31,6 @@ delete_space, ) from nemo_text_processing.text_normalization.en.utils import load_labels -from pynini.lib import pynutil def get_quantity( @@ -40,7 +41,7 @@ def get_quantity( e.g. one million -> integer_part: "1" quantity: "million" e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million" - Args: + Args: decimal: decimal FST cardinal_up_to_hundred: cardinal FST input_case: accepting either "lower_cased" or "cased" input. @@ -96,7 +97,9 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): point = pynutil.delete("point") optional_graph_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space, 0, 1, + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space, + 0, + 1, ) graph_fractional = pynutil.insert("fractional_part: \"") + graph_decimal + pynutil.insert("\"") diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/en/taggers/electronic.py index 2a2634cf4..0a41b4702 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/electronic.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path, get_various_formats from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -25,7 +27,6 @@ insert_space, ) from nemo_text_processing.text_normalization.en.utils import load_labels -from pynini.lib import pynutil class ElectronicFst(GraphFst): @@ -105,7 +106,13 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED): delete_extra_space + url_symbols + delete_extra_space - + (domain | pynini.closure(accepted_username + delete_extra_space,) + accepted_username) + + ( + domain + | pynini.closure( + accepted_username + delete_extra_space, + ) + + accepted_username + ) ) protocol_default = ( diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/en/taggers/measure.py index dcb37e5fc..69eeaa56e 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/measure.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -27,7 +29,6 @@ delete_space, get_singulars, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): @@ -57,7 +58,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPU graph_unit_plural = pynini.compose(casing_graph, graph_unit_plural).optimize() optional_graph_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space, 0, 1, + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space, + 0, + 1, ) unit_singular = convert_space(graph_unit_singular) diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/money.py b/nemo_text_processing/inverse_text_normalization/en/taggers/money.py index 5b08491c7..2c5d5ad78 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/money.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -29,7 +31,6 @@ get_singulars, insert_space, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): @@ -52,7 +53,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPU # "one fifty" -> "one hundred fifty" with_hundred = pynini.compose( pynini.closure(NEMO_NOT_SPACE) + pynini.accep(" ") + pynutil.insert("hundred ") + NEMO_SIGMA, - pynini.compose(cardinal_graph, NEMO_DIGIT ** 3), + pynini.compose(cardinal_graph, NEMO_DIGIT**3), ) cardinal_graph |= with_hundred graph_decimal_final = decimal.final_graph_wo_negative diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/en/taggers/ordinal.py index a60f704f2..632e03685 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/ordinal.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -22,7 +24,6 @@ GraphFst, capitalized_input_graph, ) -from pynini.lib import pynutil class OrdinalFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/en/taggers/punctuation.py index 10de67cdb..f53037da0 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/punctuation.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/punctuation.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class PunctuationFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/en/taggers/telephone.py index 8d0f67028..9a106ca78 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/telephone.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -28,7 +30,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil def get_serial_number(cardinal): @@ -39,7 +40,7 @@ def get_serial_number(cardinal): """ digit = pynini.compose(cardinal.graph_no_exception, NEMO_DIGIT) - two_digit = pynutil.add_weight(pynini.compose(cardinal.graph_two_digit, NEMO_DIGIT ** 2), 0.002) + two_digit = pynutil.add_weight(pynini.compose(cardinal.graph_two_digit, NEMO_DIGIT**2), 0.002) character = digit | two_digit | NEMO_ALPHA sequence = (NEMO_LOWER_NOT_A | digit) + pynini.closure(pynutil.delete(" ") + character, 2) sequence |= character + pynini.closure(pynutil.delete(" ") + (digit | NEMO_ALPHA), 2) @@ -60,7 +61,7 @@ def get_serial_number(cardinal): class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone numbers, e.g. + Finite state transducer for classifying telephone numbers, e.g. one two three one two three five six seven eight -> { number_part: "123-123-5678" } This class also support card number and IP format. @@ -115,7 +116,7 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): triple_digit.invert() # to handle cases like "one twenty three" - two_digit_cardinal = pynini.compose(cardinal.graph_no_exception, NEMO_DIGIT ** 2) + two_digit_cardinal = pynini.compose(cardinal.graph_no_exception, NEMO_DIGIT**2) double_digit_to_digit = ( pynini.compose(double_digit, str_to_digit + pynutil.delete(" ") + str_to_digit) | two_digit_cardinal ) @@ -138,7 +139,7 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): number_part = pynini.compose( single_double_or_triple_digit, - NEMO_DIGIT ** 3 + pynutil.insert("-") + NEMO_DIGIT ** 3 + pynutil.insert("-") + NEMO_DIGIT ** 4, + NEMO_DIGIT**3 + pynutil.insert("-") + NEMO_DIGIT**3 + pynutil.insert("-") + NEMO_DIGIT**4, ).optimize() number_part = pynutil.insert("number_part: \"") + number_part.optimize() + pynutil.insert("\"") @@ -155,16 +156,16 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): graph = optional_country_code + number_part # credit card number - space_four_digits = insert_space + NEMO_DIGIT ** 4 + space_four_digits = insert_space + NEMO_DIGIT**4 space_five_digits = space_four_digits + NEMO_DIGIT space_six_digits = space_five_digits + NEMO_DIGIT credit_card_graph = pynini.compose( single_double_or_triple_digit, - NEMO_DIGIT ** 4 + (space_six_digits | (space_four_digits ** 2)) + space_four_digits, + NEMO_DIGIT**4 + (space_six_digits | (space_four_digits**2)) + space_four_digits, ).optimize() credit_card_graph |= pynini.compose( - single_double_or_triple_digit, NEMO_DIGIT ** 4 + space_six_digits + space_five_digits + single_double_or_triple_digit, NEMO_DIGIT**4 + space_six_digits + space_five_digits ).optimize() graph |= pynutil.insert("number_part: \"") + credit_card_graph.optimize() + pynutil.insert("\"") @@ -172,7 +173,7 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): # SSN ssn_graph = pynini.compose( single_double_or_triple_digit, - NEMO_DIGIT ** 3 + pynutil.insert("-") + NEMO_DIGIT ** 2 + pynutil.insert("-") + NEMO_DIGIT ** 4, + NEMO_DIGIT**3 + pynutil.insert("-") + NEMO_DIGIT**2 + pynutil.insert("-") + NEMO_DIGIT**4, ).optimize() graph |= pynutil.insert("number_part: \"") + ssn_graph.optimize() + pynutil.insert("\"") diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/time.py b/nemo_text_processing/inverse_text_normalization/en/taggers/time.py index 995745331..46dc71bc8 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/time.py @@ -15,6 +15,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path, num_to_word from nemo_text_processing.text_normalization.en.graph_utils import ( @@ -27,7 +29,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -70,14 +71,32 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED): graph_minute_double = pynini.union(*labels_minute_double) @ cardinal graph_minute_verbose = pynini.cross("half", "30") | pynini.cross("quarter", "15") - oclock = pynini.cross(pynini.union("o' clock", "o clock", "o'clock", "oclock", "hundred hours",), "",) + oclock = pynini.cross( + pynini.union( + "o' clock", + "o clock", + "o'clock", + "oclock", + "hundred hours", + ), + "", + ) if input_case == INPUT_CASED: minute_to_graph = capitalized_input_graph(minute_to_graph) graph_minute_single = capitalized_input_graph(graph_minute_single) graph_minute_double = capitalized_input_graph(graph_minute_double) graph_minute_verbose |= pynini.cross("Half", "30") | pynini.cross("Quarter", "15") - oclock |= pynini.cross(pynini.union("O' clock", "O clock", "O'clock", "Oclock", "Hundred hours",), "",) + oclock |= pynini.cross( + pynini.union( + "O' clock", + "O clock", + "O'clock", + "Oclock", + "Hundred hours", + ), + "", + ) final_graph_hour = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") graph_minute = ( diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py index 0b5bdfa2b..365257b6c 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/tokenize_and_classify.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.en.taggers.date import DateFst from nemo_text_processing.inverse_text_normalization.en.taggers.decimal import DecimalFst @@ -36,7 +37,7 @@ delete_space, generator_main, ) -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -67,9 +68,9 @@ def __init__( far_file = os.path.join(cache_dir, f"en_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") cardinal = CardinalFst(input_case=input_case) cardinal_graph = cardinal.fst @@ -116,4 +117,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/en/taggers/whitelist.py index 26ef5501b..f0f20cba7 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/whitelist.py @@ -17,6 +17,8 @@ import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, @@ -26,7 +28,6 @@ string_map_cased, ) from nemo_text_processing.text_normalization.en.utils import load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/en/taggers/word.py b/nemo_text_processing/inverse_text_normalization/en/taggers/word.py index 714e68efe..2c5fdc8c2 100644 --- a/nemo_text_processing/inverse_text_normalization/en/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/en/taggers/word.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/utils.py b/nemo_text_processing/inverse_text_normalization/en/utils.py index 00b6a636f..cd54850ce 100644 --- a/nemo_text_processing/inverse_text_normalization/en/utils.py +++ b/nemo_text_processing/inverse_text_normalization/en/utils.py @@ -27,7 +27,7 @@ def num_to_word(x: Union[str, int]): Args x: integer - Returns: spoken representation + Returns: spoken representation """ if isinstance(x, int): x = str(x) @@ -41,7 +41,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py index 8d9c3a621..d9ffd5e99 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/date.py index 75a6e0d09..116bd445b 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/date.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_space, ) -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/decimal.py index 41b3a281a..60cdc138b 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/electronic.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/electronic.py index 4c9951c95..d390868ac 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class ElectronicFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/fraction.py index ca2bdcee2..780185325 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/fraction.py @@ -18,7 +18,7 @@ class FractionFst(GraphFst): """ - Finite state transducer for verbalizing fraction, + Finite state transducer for verbalizing fraction, """ def __init__(self): diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/measure.py index c5a761761..3ec608881 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/measure.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/money.py index 7218257f7..a4f7c7594 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/money.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py index 378939812..2f0a9453b 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/ordinal.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/telephone.py index 6c37ba468..141e41fe5 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/telephone.py @@ -14,15 +14,16 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class TelephoneFst(GraphFst): """ Finite state transducer for verbalizing telephone, e.g. telephone { number_part: "123-123-5678" } - -> 123-123-5678 + -> 123-123-5678 """ def __init__(self): diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/time.py index 5a2c87093..061334796 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/time.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -21,7 +23,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py index a8ea18700..86c1b575b 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/verbalize_final.py @@ -14,15 +14,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now """ diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py index 67ffe4c76..9132a639e 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py @@ -15,9 +15,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WhiteListFst(GraphFst): """ @@ -34,5 +35,5 @@ def __init__(self): + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete("\"") ) - graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/en/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/en/verbalizers/word.py index 6e94ac8c8..e75474e9b 100644 --- a/nemo_text_processing/inverse_text_normalization/en/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/en/verbalizers/word.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WordFst(GraphFst): """ @@ -28,6 +29,6 @@ def __init__(self): super().__init__(name="word", kind="verbalize") chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/data/dates/months_cased.tsv b/nemo_text_processing/inverse_text_normalization/es/data/dates/months_cased.tsv new file mode 100644 index 000000000..137183097 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/dates/months_cased.tsv @@ -0,0 +1,12 @@ +Enero +Febrero +Marzo +Abril +Mayo +Junio +Julio +Agosto +Septiembre +Octubre +Noviembre +Diciembre \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/dates/year_suffix_cased.tsv b/nemo_text_processing/inverse_text_normalization/es/data/dates/year_suffix_cased.tsv new file mode 100644 index 000000000..221fd3605 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/dates/year_suffix_cased.tsv @@ -0,0 +1,11 @@ +A. N. E. antes de nuestra era +A. E. C. antes de la era común +A. C. antes de Cristo +A. J. C. antes de Jesucristo +A. P. antes del presente +N. E. nuestra era +E. C. era común +D. C. después de Cristo +D. D. J. C. después de Jesucristo +B. C. B C +A. D. a d diff --git a/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_plural.tsv b/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_plural.tsv index e0f4284cc..1986f5cdc 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_plural.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_plural.tsv @@ -23,6 +23,12 @@ gsm g s m gsm ge ese eme psi p s i psi pe ese i -° c grados centígrados -° f grados farenheit -° k grados kelvin +° C grados centígrados +° F grados farenheit +° K grados kelvin +mb megabits +MB megabytes +gb gigabits +GB gigabytes +TB terabytes +PB petabytes \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_singular.tsv b/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_singular.tsv index 22163c72f..13f977c83 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_singular.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/measures/measurements_singular.tsv @@ -17,9 +17,9 @@ min minuto % por ciento % porciento s segundo -° c grado centígrado -° f grado farenheit -° k grado kelvin +° C grado centígrado +° F grado farenheit +° K grado kelvin mph milla por hora kph kilómetro por hora gsm gramo por metro cuadrado @@ -27,3 +27,9 @@ gsm g s m gsm ge ese eme psi p s i psi pe ese i +mb megabit +MB megabyte +gb gigabit +GB gigabyte +TB terabyte +PB petabyte \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural.tsv b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural.tsv index 43353c7ee..5d6f9a530 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural.tsv @@ -1,6 +1,7 @@ € euros US$ dólares estadounidenses US$ dólares americanos +CAD$ dólares canadienses $ dólares $ pesos ¥ yenes diff --git a/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural_capitalized.tsv b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural_capitalized.tsv new file mode 100644 index 000000000..c65809d3d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_plural_capitalized.tsv @@ -0,0 +1,75 @@ +US$ Dólares Estadounidenses +US$ dólares Estadounidenses +US$ Dólares estadounidenses +US$ Dólares Americanos +US$ dólares Americanos +US$ Dólares americanos +CAD$ dólares Canadienses +CAD$ Dólares canadienses +CAD$ Dólares Canadienses +AR$ Pesos Argentinos +AR$ pesos Argentinos +AR$ Pesos argentinos +BRL Reales Brasileños +BRL reales Brasileños +BRL Reales brasileños +CHF Francos Suizos +CHF francos Suizos +CHF Francos suizos +CLP Pesos Chilenos +CLP pesos Chilenos +CLP Pesos chilenos +CNY Yuan Chinos +CNY yuan Chinos +CNY Yuan chinos +COP Pesos Colombianos +COP pesos Colombianos +COP Pesos colombianos +CRC Colones Costarricenses +CRC colones Costarricenses +CRC Colones costarricenses +CUP Pesos Cubanos +CUP pesos Cubanos +CUP Pesos cubanos +RD$ Pesos Dominicanos +RD$ pesos Dominicanos +RD$ Pesos dominicanos +GBP Libras Esterlinas +GBP libras Esterlinas +GBP Libras esterlinas +HKD Dólares De Hong Kong +HKD dólares de Hong Kong +HKD Dólares de hong kong +INR Rupias Indias +INR rupias Indias +INR Rupias indias +Mex$ Pesos Mexicanos +Mex$ pesos Mexicanos +Mex$ Pesos mexicanos +SVC Colones Salvadoreños +SVC colones Salvadoreños +SVC Colones salvadoreños +UYU Pesos Uruguayos +UYU pesos Uruguayos +UYU Pesos uruguayos +VES Bolívares Soberanos +VES bolívares Soberanos +VES Bolívares soberanos +BOP Pesos Bolivianos +BOP pesos Bolivianos +BOP Pesos bolivianos +CLE Escudos Chilenos +CLE escudos Chilenos +CLE Escudos chilenos +ECS Sucres Ecuatorianos +ECS sucres Ecuatorianos +ECS Sucres ecuatorianos +PEH Soles De Oro +PEH soles de Oro +PEH Soles de oro +VEB Bolívares Venezolanos +VEB bolívares Venezolanos +VEB Bolívares venezolanos +VEF Bolívares Fuertes +VEF bolívares Fuertes +VEF Bolívares fuertes \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular.tsv b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular.tsv index 879c8f1fe..3532bb2dd 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular.tsv @@ -1,6 +1,7 @@ € euro US$ dólar estadounidense US$ dólar americano +CAD$ dólar canadiense $ dólar $ peso ¥ yen diff --git a/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular_capitalized.tsv b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular_capitalized.tsv new file mode 100644 index 000000000..7e8ba2611 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/money/currency_major_singular_capitalized.tsv @@ -0,0 +1,76 @@ +US$ dólar Estadounidense +US$ Dólar Estadounidense +US$ Dólar estadounidense +US$ dólar Estadounidense +US$ dólar Americano +US$ Dólar Americano +US$ Dólar americano +CAD$ dólar Canadiense +CAD$ Dólar canadiense +CAD$ Dólar Canadiense +AR$ peso Argentino +AR$ Peso Argentino +AR$ Peso argentino +BRL real Brasileño +BRL Real Brasileño +BRL Real brasileño +CHF franco Suizo +CHF Franco Suizo +CHF Franco suizo +CLP Peso Chileno +CLP peso Chileno +CLP Peso chileno +CNY Yuan Chino +CNY yuan Chino +CNY Yuan chino +COP Peso Colombiano +COP peso Colombiano +COP Peso colombiano +CRC Colón Costarricense +CRC colón Costarricense +CRC Colón costarricense +CUP Peso Cubano +CUP peso Cubano +CUP Peso cubano +RD$ Peso Dominicano +RD$ peso Dominicano +RD$ Peso dominicano +GBP Libra Esterlina +GBP libra Esterlina +GBP Libra esterlina +HKD Dólar De Hong Kong +HKD dólar de Hong Kong +HKD Dólar de hong kong +INR Rupia India +INR rupia India +INR Rupia india +Mex$ Peso Mexicano +Mex$ peso Mexicano +Mex$ Peso mexicano +SVC Colón Salvadoreño +SVC colón Salvadoreño +SVC Colón salvadoreño +UYU Peso Uruguayo +UYU peso Uruguayo +UYU Peso uruguayo +VES Bolívar Soberano +VES bolívar Soberano +VES Bolívar soberano +BOP Peso Boliviano +BOP peso Boliviano +BOP Peso boliviano +CLE Escudo Chileno +CLE escudo Chileno +CLE Escudo chileno +ECS Sucre Ecuatoriano +ECS sucre Ecuatoriano +ECS Sucre ecuatoriano +PEH Sol De Oro +PEH sol de Oro +PEH Sol de oro +VEB Bolívar Venezolano +VEB bolívar Venezolano +VEB Bolívar venezolano +VEF Bolívar Fuerte +VEF bolívar Fuerte +VEF Bolívar fuerte \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/ordinals/digit_capitalized.tsv b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/digit_capitalized.tsv new file mode 100644 index 000000000..459ea85e1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/digit_capitalized.tsv @@ -0,0 +1,22 @@ +Primero uno +Primera uno +Primer uno +Segundo dos +Segunda dos +Tercero tres +Tercera tres +Tercer tres +Cuarto cuatro +Cuarta cuatro +Quinto cinco +Quinta cinco +Sexto seis +Sexta seis +Séptimo siete +Séptima siete +Sétimo siete +Sétima siete +Octavo ocho +Octava ocho +Noveno nueve +Novena nueve diff --git a/nemo_text_processing/inverse_text_normalization/es/data/ordinals/hundreds_capitalized.tsv b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/hundreds_capitalized.tsv new file mode 100644 index 000000000..0172f8f63 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/hundreds_capitalized.tsv @@ -0,0 +1,18 @@ +Centésimo ciento +Centésima ciento +Ducentésimo doscientos +Ducentésima doscientos +Tricentésimo trescientos +Tricentésima trescientos +Cuadringentésimo cuatrocientos +Cuadringentésima cuatrocientos +Quingentésimo quinientos +Quingentésima quinientos +Sexcentésimo seiscientos +Sexcentésima seiscientos +Septingentésimo setecientos +Septingentésima setecientos +Octingentésimo ochocientos +Octingentésima ochocientos +Noningentésimo novecientos +Noningentésima novecientos diff --git a/nemo_text_processing/inverse_text_normalization/es/data/ordinals/teen_capitalized.tsv b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/teen_capitalized.tsv new file mode 100644 index 000000000..80f012ba5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/teen_capitalized.tsv @@ -0,0 +1,60 @@ +Décimo diez +Décima diez +Decimoprimero once +Decimoprimera once +Decimoprimer once +Décimo Primero once +Décima Primera once +Décimo Primera once +Décimo Primer once +Undécimo once +Undécima once +Decimosegundo doce +Decimosegunda doce +Décimo Segundo doce +Décima Segunda doce +Décimo Segunda doce +Duodécimo doce +Duodécima doce +Decimotercero trece +Decimotercera trece +Decimotercer trece +Décimo Tercero trece +Décima Tercera trece +Décimo Tercera trece +Décimo Tercer trece +Decimocuarto catorce +Decimocuarta catorce +Décimo Cuarto catorce +Décima Cuarta catorce +Décimo Cuarta catorce +Decimoquinto quince +Decimoquinta quince +Décimo Quinto quince +Décima Quinta quince +Décimo Quinta quince +Decimosexto dieciséis +Decimosexta dieciséis +Décimo Sexto dieciséis +Décima Sexta dieciséis +Décimo Sexta dieciséis +Decimoséptimo diecisiete +Decimoséptima diecisiete +Décimo Séptimo diecisiete +Décima Séptima diecisiete +Décimo Séptima diecisiete +Décimo Sétimo diecisiete +Décimo Sétima diecisiete +Décima Sétima diecisiete +Decimosétimo diecisiete +Decimosétima diecisiete +Decimoctavo dieciocho +Decimoctava dieciocho +Décimo Octavo dieciocho +Décima Octava dieciocho +Décimo Octava dieciocho +Decimonoveno diecinueve +Decimonovena diecinueve +Décimo Noveno diecinueve +Décima Novena diecinueve +Décimo Novena diecinueve diff --git a/nemo_text_processing/inverse_text_normalization/es/data/ordinals/ties_capitalized.tsv b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/ties_capitalized.tsv new file mode 100644 index 000000000..58e0eff28 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/ties_capitalized.tsv @@ -0,0 +1,15 @@ +Vigésimo veinte +Vigésima veinte +Trigésimo treinta +Cuadragésimo cuarenta +Cuadragésima cuarenta +Quincuagésimo cincuenta +Quincuagésima cincuenta +Sexagésimo sesenta +Sexagésima sesenta +Septuagésimo setenta +Septuagésima setenta +Octogésimo ochenta +Octogésima ochenta +Nonagésimo noventa +Nonagésima noventa diff --git a/nemo_text_processing/inverse_text_normalization/es/data/ordinals/twenties_capitalized.tsv b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/twenties_capitalized.tsv new file mode 100644 index 000000000..40e73e815 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/ordinals/twenties_capitalized.tsv @@ -0,0 +1,50 @@ +Vigesimoprimero veintiuno +Vigesimoprimera veintiuno +Vigesimoprimer veintiuno +Vigésimo Primero veintiuno +Vigésimo Primera veintiuno +Vigésima Primera veintiuno +Vigésimo Primer veintiuno +Vigesimosegundo veintidós +Vigesimosegunda veintidós +Vigésimo Segundo veintidós +Vigésimo Segunda veintidós +Vigésima Segunda veintidós +Vigesimotercero veintitrés +Vigesimotercera veintitrés +Vigesimotercer veintitrés +Vigésimo Tercero veintitrés +Vigésimo Tercera veintitrés +Vigésima Tercera veintitrés +Vigésimo Tercer veintitrés +Vigesimocuarto veinticuatro +Vigesimocuarta veinticuatro +Vigésimo Cuarto veinticuatro +Vigésimo Cuarta veinticuatro +Vigésima Cuarta veinticuatro +Vigesimoquinto veinticinco +Vigesimoquinta veinticinco +Vigésimo Quinto veinticinco +Vigésimo Quinta veinticinco +Vigésima Quinta veinticinco +Vigesimosexto veintiséis +Vigesimosexta veintiséis +Vigésimo Sexto veintiséis +Vigésimo Sexta veintiséis +Vigésima Sexta veintiséis +Vigesimoséptimo veintisiete +Vigesimoséptima veintisiete +Vigésimo Séptimo veintisiete +Vigésimo Séptima veintisiete +Vigésima Séptima veintisiete +Vigesimoctavo veintiocho +Vigesimoctava veintiocho +Vigesimooctavo veintiocho +Vigesimooctava veintiocho +Vigésimo Octavo veintiocho +Vigésimo Octava veintiocho +Vigésima Octava veintiocho +Vigesimonoveno veintinueve +Vigesimonovena veintinueve +Vigésimo Noveno veintinueve +Vigésimo Novena veintinueve diff --git a/nemo_text_processing/inverse_text_normalization/es/data/roman/digit.tsv b/nemo_text_processing/inverse_text_normalization/es/data/roman/digit.tsv index 0610b4a54..e5fde2cc6 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/roman/digit.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/roman/digit.tsv @@ -1,9 +1,9 @@ -i 1 -ii 2 -iii 3 -iv 4 -v 5 -vi 6 -vii 7 -viii 8 -ix 9 \ No newline at end of file +I 1 +II 2 +III 3 +IV 4 +V 5 +VI 6 +VII 7 +VIII 8 +IX 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/roman/hundreds.tsv b/nemo_text_processing/inverse_text_normalization/es/data/roman/hundreds.tsv index cdbdb6814..5e04779be 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/roman/hundreds.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/roman/hundreds.tsv @@ -1,9 +1,9 @@ -c 1 -cc 2 -ccc 3 -cd 4 -d 5 -dc 6 -dcc 7 -dccc 8 -cm 9 \ No newline at end of file +C 1 +CC 2 +CCC 3 +CD 4 +D 5 +DC 6 +DCC 7 +DCCC 8 +CM 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/roman/thousands.tsv b/nemo_text_processing/inverse_text_normalization/es/data/roman/thousands.tsv index 19e96b9c6..164689802 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/roman/thousands.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/roman/thousands.tsv @@ -1,3 +1,3 @@ -m 1 -mm 2 -mmm 3 +M 1 +MM 2 +MMM 3 diff --git a/nemo_text_processing/inverse_text_normalization/es/data/roman/ties.tsv b/nemo_text_processing/inverse_text_normalization/es/data/roman/ties.tsv index ac043aa14..445773d91 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/roman/ties.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/roman/ties.tsv @@ -1,9 +1,9 @@ -x 1 -xx 2 -xxx 3 -xl 4 -l 5 -lx 6 -lxx 7 -lxxx 8 -xc 9 \ No newline at end of file +X 1 +XX 2 +XXX 3 +XL 4 +L 5 +LX 6 +LXX 7 +LXXX 8 +XC 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/time/time_suffix_cased.tsv b/nemo_text_processing/inverse_text_normalization/es/data/time/time_suffix_cased.tsv new file mode 100644 index 000000000..b04bd5193 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/time/time_suffix_cased.tsv @@ -0,0 +1,15 @@ +Peme P.M. +Pe Eme P.M. +P M P.M. +PM P.M. +P.M. +p.M P.M. +Ame A.M +A Eme A.M +AM A.M +A.M +A.M A.M +A M A.M +de la tarde P.M. +de la noche P.M. +de la mañana A.M diff --git a/nemo_text_processing/inverse_text_normalization/es/data/time/time_zone.tsv b/nemo_text_processing/inverse_text_normalization/es/data/time/time_zone.tsv index 6c86a3e4e..55f0297b0 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/time/time_zone.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/time/time_zone.tsv @@ -1,42 +1,42 @@ -utc u t c -cst c s t -cet c e t -pst p s t -est e s t -mdt m d t -mst m s t -pt p t -et e t -mt m t -gmt g m t -adt hora de verano del atlántico -amt hora estándar del amazonas -art hora estándar de argentina -ast hora estándar del atlántico -bot hora de bolivia -brt hora estándar de brasilia -clst hora de verano de chile -clt hora estándar de chile -cot hora estándar de colombia -east hora estándar de la isla de pascua -ect hora de ecuador -eeast hora de verano de la isla de pascua -eest hora de verano de europa oriental -eet hora estándar de europa oriental -fkst hora de verano de las malvinas -fnt hora estándar de fernando de noronha -galt hora de galápagos -gft hora de la guayana francesa -gyt hora de guyana -hkt hora estándar de hong kong -jst hora estándar de japón -kst hora estándar de corea -pet hora estándar de perú -pyst hora de verano de paraguay -pyt hora estándar de paraguay -sgt hora de singapur -uyst hora de verano de uruguay -uyt hora de uruguay -vet hora de venezuela -west hora de verano de europa oriental -wet hora estándar de europa oriental +UTC u t c +CST c s t +CET c e t +PST p s t +EST e s t +MDT m d t +MST m s t +PT p t +ET e t +MT m t +GMT g m t +ADT hora de verano del atlántico +AMT hora estándar del amazonas +ART hora estándar de argentina +AST hora estándar del atlántico +BOT hora de bolivia +BRT hora estándar de brasilia +CLST hora de verano de chile +CLT hora estándar de chile +COT hora estándar de colombia +EAST hora estándar de la isla de pascua +ECT hora de ecuador +EEAST hora de verano de la isla de pascua +EEST hora de verano de europa oriental +EET hora estándar de europa oriental +FKST hora de verano de las malvinas +FNT hora estándar de fernando de noronha +GALT hora de galápagos +GFT hora de la guayana francesa +GYT hora de guyana +HKT hora estándar de hong kong +JST hora estándar de japón +KST hora estándar de corea +PET hora estándar de perú +PYST hora de verano de paraguay +PYT hora estándar de paraguay +SGT hora de singapur +UYST hora de verano de uruguay +UYT hora de uruguay +VET hora de venezuela +WEST hora de verano de europa oriental +WET hora estándar de europa oriental \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/time/time_zone_cased.tsv b/nemo_text_processing/inverse_text_normalization/es/data/time/time_zone_cased.tsv new file mode 100644 index 000000000..e635698d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/es/data/time/time_zone_cased.tsv @@ -0,0 +1,42 @@ +UTC U T C +CST C S T +CET C E T +PST P S T +EST E S T +MDT M D T +MST M S T +PT P T +ET E T +MT M T +GMT G M T +ADT Hora de Verano del Atlántico +AMT Hora Estándar del Amazonas +ART Hora Estándar de Argentina +AST Hora Estándar del Atlántico +BOT Hora de Bolivia +BRT Hora Estándar de Brasilia +CLST Hora de Verano de Chile +CLT Hora Estándar de Chile +COT Hora Estándar de Colombia +EAST Hora Estándar de la Isla de Pascua +ECT Hora de Ecuador +EEAST Hora de Verano de la Isla de Pascua +EEST Hora de Verano de Europa Oriental +EET Hora Estándar de Europa Oriental +FKST Hora de Verano de Las Malvinas +FNT Hora Estándar de Fernando de Noronha +GALT Hora de Galápagos +GFT Hora de la Guayana Francesa +GYT Hora de Guyana +HKT Hora Estándar de Hong Kong +JST Hora Estándar de Japón +KST Hora Estándar de Corea +PET Hora Estándar de Perú +PYST Hora de Verano de Paraguay +PYT Hora Estándar de Paraguay +SGT Hora de Singapur +UYST Hora de Verano de Uruguay +UYT Hora de Uruguay +VET Hora de Venezuela +WEST Hora de Verano de Europa Oriental +WET Hora Estándar de Europa Oriental \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/data/whitelist.tsv b/nemo_text_processing/inverse_text_normalization/es/data/whitelist.tsv index 60253820a..3973c1a0c 100644 --- a/nemo_text_processing/inverse_text_normalization/es/data/whitelist.tsv +++ b/nemo_text_processing/inverse_text_normalization/es/data/whitelist.tsv @@ -1,16 +1,47 @@ -ud. usted -uds. ustedes -vd. vosotros -vds. vosotros -dr. doctor -dra. doctora -d. don -da. doña -ee. uu. estados unidos +Ud. usted +Uds. ustedes +Vd. vosotros +Vds. vosotros +Dr. doctor +Dra. doctora +D. don +Da. doña +EE. UU. estados unidos p.ej. por ejemplo -prof. profesor -profa. profesora -sr. señor -sra. señora -srta. señorita +Prof. profesor +Profa. profesora +Sr. señor +Sra. señora +Sres. señores +Srta. señorita etc. etcétera +TXT t x t +TXT T x t +TXT T X T +GPS g p s +GPS G p s +GPS G P S +DNI d n i +DNI D n i +DNI D N I +ISSN I S S N +ISSN I s s n +ISSN i s s n +SMS S M S +SMS S m s +SMS s m s +ISBN I S B N +ISBN I s b n +ISBN i s b n +URL U R L +URL U r l +URL u r l +RPG R P G +RPG R p g +RPG r p g +ADN A D N +ADN A d n +ADN a d n +JPG J P G +JPG J p g +JPG j p g \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es/graph_utils.py b/nemo_text_processing/inverse_text_normalization/es/graph_utils.py index 164ebcdbb..0396b5b4c 100644 --- a/nemo_text_processing/inverse_text_normalization/es/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/es/graph_utils.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.es.utils import get_abs_path from pynini.lib import pynutil +from nemo_text_processing.text_normalization.es.utils import get_abs_path + def int_to_roman(fst: 'pynini.FstLike') -> 'pynini.FstLike': """ diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/es/taggers/cardinal.py index 07a1e8316..d3082509a 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/cardinal.py @@ -14,18 +14,27 @@ import pynini -from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + capitalized_input_graph, + delete_space, +) +from nemo_text_processing.text_normalization.es.graph_utils import ES_MINUS class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals - e.g. menos veintitrés -> cardinal { negative: "-" integer: "23"} + e.g. menos veintitrés -> cardinal { negative: "-" integer: "23"} This class converts cardinals up to (but not including) "un cuatrillón", i.e up to "one septillion" in English (10^{24}). - Cardinals below ten are not converted (in order to avoid + Cardinals below ten are not converted (in order to avoid "vivo en una casa" --> "vivo en 1 casa" and any other odd conversions.) Although technically Spanish grammar requires that "y" only comes after @@ -34,10 +43,15 @@ class CardinalFst(GraphFst): inside cardinal numbers). e.g. "mil y una" -> cardinal { integer: "1001"} e.g. "ciento y una" -> cardinal { integer: "101"} + + Args: + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self): + def __init__(self, input_case: str = INPUT_LOWER_CASED): super().__init__(name="cardinal", kind="classify") + self.input_case = input_case + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) @@ -46,7 +60,7 @@ def __init__(self): graph_hundreds = pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")) full_graph_ties = (graph_ties | pynutil.insert("0")) + ( - (delete_space + pynutil.delete("y") + delete_space + graph_digit) | pynutil.insert("0") + (delete_space + self.delete_word("y") + delete_space + graph_digit) | pynutil.insert("0") ) graph_hundred_component = graph_hundreds | pynutil.insert("0") @@ -60,27 +74,27 @@ def __init__(self): ) self.graph_hundred_component_at_least_one_none_zero_digit = ( graph_hundred_component_at_least_one_none_zero_digit - ) + ).optimize() graph_thousands = pynini.union( - graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("mil"), - pynutil.insert("001") + pynutil.delete("mil"), # because we say 'mil', not 'un mil' + graph_hundred_component_at_least_one_none_zero_digit + delete_space + self.delete_word("mil"), + pynutil.insert("001") + self.delete_word("mil"), # because we say 'mil', not 'un mil' pynutil.insert("000", weight=0.1), ) graph_millones = pynini.union( graph_hundred_component_at_least_one_none_zero_digit + delete_space - + (pynutil.delete("millones") | pynutil.delete("millón")), - pynutil.insert("000") + pynutil.delete("millones"), # to allow for 'mil millones' + + (self.delete_word("millones") | self.delete_word("millón")), + pynutil.insert("000") + self.delete_word("millones"), # to allow for 'mil millones' ) graph_mil_millones = pynini.union( - graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("mil"), - pynutil.insert("001") + pynutil.delete("mil"), # because we say 'mil', not 'un mil' + graph_hundred_component_at_least_one_none_zero_digit + delete_space + self.delete_word("mil"), + pynutil.insert("001") + self.delete_word("mil"), # because we say 'mil', not 'un mil' ) graph_mil_millones += delete_space + ( - graph_millones | pynutil.insert("000") + pynutil.delete("millones") + graph_millones | pynutil.insert("000") + self.delete_word("millones") ) # allow for 'mil millones' graph_mil_millones |= pynutil.insert("000000", weight=0.1) @@ -88,36 +102,36 @@ def __init__(self): graph_millardo = ( graph_hundred_component_at_least_one_none_zero_digit + delete_space - + (pynutil.delete("millardo") | pynutil.delete("millardos")) + + (self.delete_word("millardo") | self.delete_word("millardos")) ) graph_billones = pynini.union( graph_hundred_component_at_least_one_none_zero_digit + delete_space - + (pynutil.delete("billones") | pynutil.delete("billón")), + + (self.delete_word("billones") | self.delete_word("billón")), ) graph_mil_billones = pynini.union( - graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("mil"), - pynutil.insert("001") + pynutil.delete("mil"), # because we say 'mil', not 'un mil' + graph_hundred_component_at_least_one_none_zero_digit + delete_space + self.delete_word("mil"), + pynutil.insert("001") + self.delete_word("mil"), # because we say 'mil', not 'un mil' ) graph_mil_billones += delete_space + ( - graph_billones | pynutil.insert("000") + pynutil.delete("billones") + graph_billones | pynutil.insert("000") + self.delete_word("billones") ) # allow for 'mil billones' graph_mil_billones |= pynutil.insert("000000", weight=0.1) graph_trillones = pynini.union( graph_hundred_component_at_least_one_none_zero_digit + delete_space - + (pynutil.delete("trillones") | pynutil.delete("trillón")), + + (self.delete_word("trillones") | self.delete_word("trillón")), ) graph_mil_trillones = pynini.union( - graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("mil"), - pynutil.insert("001") + pynutil.delete("mil"), # because we say 'mil', not 'un mil' + graph_hundred_component_at_least_one_none_zero_digit + delete_space + self.delete_word("mil"), + pynutil.insert("001") + self.delete_word("mil"), # because we say 'mil', not 'un mil' ) graph_mil_trillones += delete_space + ( - graph_trillones | pynutil.insert("000") + pynutil.delete("trillones") + graph_trillones | pynutil.insert("000") + self.delete_word("trillones") ) # allow for 'mil trillones' graph_mil_trillones |= pynutil.insert("000000", weight=0.1) @@ -143,35 +157,47 @@ def __init__(self): pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), "0" ) - self.graph_no_exception = graph + self.graph_no_exception = graph.optimize() # save self.numbers_up_to_thousand for use in DecimalFst - digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT ** 2) | (NEMO_DIGIT ** 3) - numbers_up_to_thousand = pynini.compose(graph, digits_up_to_thousand).optimize() - self.numbers_up_to_thousand = numbers_up_to_thousand + digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3) + numbers_up_to_thousand = pynini.compose(self.graph_no_exception, digits_up_to_thousand).optimize() + self.numbers_up_to_thousand = numbers_up_to_thousand.optimize() # save self.numbers_up_to_million for use in DecimalFst digits_up_to_million = ( - NEMO_DIGIT - | (NEMO_DIGIT ** 2) - | (NEMO_DIGIT ** 3) - | (NEMO_DIGIT ** 4) - | (NEMO_DIGIT ** 5) - | (NEMO_DIGIT ** 6) + NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3) | (NEMO_DIGIT**4) | (NEMO_DIGIT**5) | (NEMO_DIGIT**6) ) numbers_up_to_million = pynini.compose(graph, digits_up_to_million).optimize() - self.numbers_up_to_million = numbers_up_to_million + self.numbers_up_to_million = numbers_up_to_million.optimize() + + if input_case == INPUT_CASED: + graph |= capitalized_input_graph(graph) + graph_digit |= capitalized_input_graph(graph_digit) + graph_zero |= capitalized_input_graph(graph_zero) + # graph_exception = capitalized_input_graph(graph_exception) + self.graph_no_exception |= capitalized_input_graph(self.graph_no_exception).optimize() + self.numbers_up_to_thousand |= capitalized_input_graph(self.numbers_up_to_thousand).optimize() # don't convert cardinals from zero to nine inclusive graph_exception = pynini.project(pynini.closure(NEMO_SPACE, 0, 1) + (graph_digit | graph_zero), 'input') - self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + self.graph = ((pynini.project(graph, "input") - graph_exception.arcsort()) @ graph).optimize() optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("menos", "\"-\"") + NEMO_SPACE, 0, 1 + pynutil.insert("negative: ") + pynini.cross(ES_MINUS, "\"-\"") + NEMO_SPACE, 0, 1 ) final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() + + def delete_word(self, word: str): + """Capitalizes word for `cased` input""" + delete_graph = pynutil.delete(word).optimize() + if self.input_case == INPUT_CASED: + if len(word) > 0: + delete_graph |= pynutil.delete(word[0].upper() + word[1:]) + + return delete_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/date.py b/nemo_text_processing/inverse_text_normalization/es/taggers/date.py index 3100e6a50..66281d225 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/date.py @@ -13,20 +13,31 @@ # limitations under the License. import pynini +from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.es.graph_utils import int_to_roman from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + GraphFst, + capitalized_input_graph, + delete_extra_space, + delete_space, +) class DateFst(GraphFst): """ - Finite state transducer for classifying date, + Finite state transducer for classifying date, e.g. primero de enero -> date { day: "1" month: "enero" } e.g. uno de enero -> date { day: "1" month: "enero" } + + Args: + cardinal: CardinalFst + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self, cardinal: GraphFst): + def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): super().__init__(name="date", kind="classify") graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) @@ -37,6 +48,10 @@ def __init__(self, cardinal: GraphFst): graph_month = pynini.string_file(get_abs_path("data/dates/months.tsv")) graph_suffix = pynini.string_file(get_abs_path("data/dates/year_suffix.tsv")).invert() + if input_case == INPUT_CASED: + graph_month |= pynini.string_file(get_abs_path("data/dates/months_cased.tsv")) + graph_suffix |= pynini.string_file(get_abs_path("data/dates/year_suffix_cased.tsv")).invert() + graph_1_to_100 = pynini.union( graph_digit, graph_twenties, @@ -67,5 +82,9 @@ def __init__(self, cardinal: GraphFst): final_graph = graph_dm | roman_centuries_graph | year_with_suffix_graph final_graph += pynutil.insert(" preserve_order: true") + + if input_case == INPUT_CASED: + final_graph |= capitalized_input_graph(final_graph) + final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/es/taggers/decimal.py index bdbf18049..8bfa560d2 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/decimal.py @@ -13,31 +13,41 @@ # limitations under the License. import pynini +from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + MIN_NEG_WEIGHT, NEMO_DIGIT, + NEMO_SIGMA, + TO_LOWER, GraphFst, + capitalized_input_graph, delete_extra_space, delete_space, ) -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.es.graph_utils import ES_MINUS -def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_million: 'pynini.FstLike') -> 'pynini.FstLike': +def get_quantity( + decimal: 'pynini.FstLike', cardinal_up_to_million: 'pynini.FstLike', input_case: str = INPUT_LOWER_CASED +) -> 'pynini.FstLike': """ Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, e.g. one million -> integer_part: "1" quantity: "million" e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million" - Args: + Args: decimal: decimal FST cardinal_up_to_million: cardinal FST + input_case: accepting either "lower_cased" or "cased" input. """ numbers = cardinal_up_to_million @ ( pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) ) - suffix = pynini.union( + suffix_labels = [ "millón", "millones", "millardo", @@ -48,7 +58,12 @@ def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_million: 'pynini.FstL "trillones", "cuatrillón", "cuatrillones", - ) + ] + suffix = pynini.union(*suffix_labels) + + if input_case == INPUT_CASED: + suffix |= pynini.union(*[x[0].upper() + x[1:] for x in suffix_labels]).optimize() + res = ( pynutil.insert("integer_part: \"") + numbers @@ -72,29 +87,34 @@ class DecimalFst(GraphFst): This decimal rule assumes that decimals can be pronounced as: (a cardinal) + ('coma' or 'punto') plus (any sequence of cardinals <1000, including 'zero') - Also writes large numbers in shortened form, e.g. + Also writes large numbers in shortened form, e.g. e.g. uno coma dos seis millón -> decimal { negative: "false" integer_part: "1" morphosyntactic_features: "," fractional_part: "26" quantity: "millón" } e.g. dos millones -> decimal { negative: "false" integer_part: "2" quantity: "millones" } e.g. mil ochocientos veinticuatro millones -> decimal { negative: "false" integer_part: "1824" quantity: "millones" } Args: cardinal: CardinalFst + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self, cardinal: GraphFst): + def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): super().__init__(name="decimal", kind="classify") # number after decimal point can be any series of cardinals <1000, including 'zero' graph_decimal = cardinal.numbers_up_to_thousand graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal - self.graph = graph_decimal + self.graph = graph_decimal.optimize() # decimal point can be denoted by 'coma' or 'punto' decimal_point = pynini.cross("coma", "morphosyntactic_features: \",\"") decimal_point |= pynini.cross("punto", "morphosyntactic_features: \".\"") + if input_case == INPUT_CASED: + decimal_point |= pynini.cross("Coma", "morphosyntactic_features: \",\"") + decimal_point |= pynini.cross("Punto", "morphosyntactic_features: \".\"") + optional_graph_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("menos", "\"true\"") + delete_extra_space, 0, 1 + pynutil.insert("negative: ") + pynini.cross(ES_MINUS, "\"true\"") + delete_extra_space, 0, 1 ) graph_fractional = pynutil.insert("fractional_part: \"") + graph_decimal + pynutil.insert("\"") @@ -109,9 +129,21 @@ def __init__(self, cardinal: GraphFst): ) final_graph = optional_graph_negative + final_graph_wo_sign - self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( - final_graph_wo_sign, cardinal.numbers_up_to_million + self.final_graph_wo_negative = ( + final_graph_wo_sign + | get_quantity(final_graph_wo_sign, cardinal.numbers_up_to_million, input_case=input_case).optimize() ) - final_graph |= optional_graph_negative + get_quantity(final_graph_wo_sign, cardinal.numbers_up_to_million) + + # accept semiotic spans that start with a capital letter + self.final_graph_wo_negative |= pynutil.add_weight( + pynini.compose(TO_LOWER + NEMO_SIGMA, self.final_graph_wo_negative), MIN_NEG_WEIGHT + ).optimize() + + quantity_graph = get_quantity(final_graph_wo_sign, cardinal.numbers_up_to_million, input_case=input_case) + final_graph |= optional_graph_negative + quantity_graph + + if input_case == INPUT_CASED: + final_graph |= capitalized_input_graph(final_graph) + final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/es/taggers/electronic.py index 53b6b4d09..a7d767119 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/electronic.py @@ -13,10 +13,21 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, GraphFst, insert_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.en.utils import get_various_formats +from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + MIN_POS_WEIGHT, + NEMO_ALPHA, + GraphFst, + capitalized_input_graph, + insert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels + class ElectronicFst(GraphFst): """ @@ -25,22 +36,36 @@ class ElectronicFst(GraphFst): and URLS (which get converted to a "protocol" field). e.g. c d f uno arroba a b c punto e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } } e.g. doble ve doble ve doble ve a b c punto e d u -> tokens { electronic { protocol: "www.abc.edu" } } + + Args: + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self): + def __init__(self, input_case: str = INPUT_LOWER_CASED): super().__init__(name="electronic", kind="classify") delete_extra_space = pynutil.delete(" ") - alpha_num = ( - NEMO_ALPHA - | pynini.string_file(get_abs_path("data/numbers/digit.tsv")) - | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + num = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) | pynini.string_file( + get_abs_path("data/numbers/zero.tsv") ) + if input_case == INPUT_CASED: + num = capitalized_input_graph(num) + + alpha_num = (NEMO_ALPHA | num).optimize() symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert() + if input_case == INPUT_CASED: + symbols = capitalized_input_graph(symbols) accepted_username = alpha_num | symbols - process_dot = pynini.cross("punto", ".") + dot = pynini.accep("punto") + if input_case == INPUT_CASED: + dot |= pynini.accep("Punto") + process_dot = pynini.cross(dot, ".") + alternative_dot = ( + pynini.closure(delete_extra_space, 0, 1) + pynini.accep(".") + pynini.closure(delete_extra_space, 0, 1) + ) username = ( pynutil.insert("username: \"") + alpha_num @@ -50,25 +75,60 @@ def __init__(self): + pynutil.insert("\"") ) single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num - server = single_alphanum | pynini.string_file(get_abs_path("data/electronic/server_name.tsv")).invert() - domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv")).invert() + + server_names = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")).invert() + if input_case == INPUT_CASED: + server_names = capitalized_input_graph(server_names) + server = single_alphanum | server_names | pynini.closure(NEMO_ALPHA, 2) + + if input_case == INPUT_CASED: + domain = [] + # get domain formats + for d in load_labels(get_abs_path("data/electronic/domain.tsv")): + domain.extend(get_various_formats(d[0])) + domain = pynini.string_map(domain).optimize() + else: + domain = pynini.string_file(get_abs_path("data/electronic/domain.tsv")).invert() + + domain = pynutil.add_weight(single_alphanum, weight=-0.0001) | domain | pynini.closure(NEMO_ALPHA, 2) + domain_graph = ( pynutil.insert("domain: \"") + server - + delete_extra_space - + process_dot - + delete_extra_space + + ((delete_extra_space + process_dot + delete_extra_space) | alternative_dot) + domain + pynutil.insert("\"") ) - graph = ( - username + delete_extra_space + pynutil.delete("arroba") + insert_space + delete_extra_space + domain_graph - ) + + at = pynini.accep("arroba") + if input_case == INPUT_CASED: + at |= pynini.accep("Arroba") + + graph = username + delete_extra_space + pynutil.delete(at) + insert_space + delete_extra_space + domain_graph ############# url ### - protocol_end = pynini.cross(pynini.union("www", "w w w", "doble ve doble ve doble ve"), "www") - protocol_start = pynini.cross(pynini.union("http", "h t t p", "hache te te pe"), "http") - protocol_start |= pynini.cross(pynini.union("https", "h t t p s", "hache te te pe ese"), "https") + if input_case == INPUT_CASED: + spoken_ws = pynini.union( + "doble ve doble ve doble ve", "Doble Ve Doble Ve Doble Ve", "Doble ve doble ve doble ve" + ) + protocol_end = pynini.cross(pynini.union(*get_various_formats("www")) | spoken_ws, "www") + + spoken_http = pynini.union("hache te te pe", "Hache te te pe", "Hache Te Te Pe") + spoken_https = pynini.union("hache te te pe ese", "Hache te te pe ese", "Hache Te Te Pe Ese") + protocol_start = pynini.cross( + pynini.union(*get_various_formats("http")) | spoken_http, "http" + ) | pynini.cross(pynini.union(*get_various_formats("https")) | spoken_https, "https") + else: + protocol_end = pynutil.add_weight( + pynini.cross(pynini.union("www", "w w w", "doble ve doble ve doble ve"), "www"), MIN_POS_WEIGHT + ) + protocol_start = pynutil.add_weight( + pynini.cross(pynini.union("http", "h t t p", "hache te te pe"), "http"), MIN_POS_WEIGHT + ) + protocol_start |= pynutil.add_weight( + pynini.cross(pynini.union("https", "h t t p s", "hache te te pe ese"), "https"), MIN_POS_WEIGHT + ) + protocol_start += pynini.cross(" dos puntos barra barra ", "://") # e.g. .com, .es @@ -76,21 +136,44 @@ def __init__(self): delete_extra_space + symbols + delete_extra_space - + (domain | pynini.closure(accepted_username + delete_extra_space,) + accepted_username) + + ( + domain + | pynini.closure( + accepted_username + delete_extra_space, + ) + + accepted_username + ) ) + protocol_default = ( + ( + (pynini.closure(delete_extra_space + accepted_username, 1) | server) + | pynutil.add_weight(pynini.closure(NEMO_ALPHA, 1), weight=0.001) + ) + + pynini.closure(ending, 1) + ).optimize() + protocol = ( pynini.closure(protocol_start, 0, 1) + protocol_end + delete_extra_space + process_dot + delete_extra_space - + (pynini.closure(delete_extra_space + accepted_username, 1) | server) - + pynini.closure(ending, 1) - ) + + protocol_default + ).optimize() + + if input_case == INPUT_CASED: + protocol |= ( + pynini.closure(protocol_start, 0, 1) + protocol_end + alternative_dot + protocol_default + ).optimize() + + protocol |= pynini.closure(protocol_end + delete_extra_space + process_dot, 0, 1) + protocol_default + protocol = pynutil.insert("protocol: \"") + protocol + pynutil.insert("\"") graph |= protocol - ######## + + if input_case == INPUT_CASED: + graph = capitalized_input_graph(graph, capitalized_graph_weight=MIN_POS_WEIGHT) final_graph = self.add_tokens(graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/es/taggers/fraction.py index f31d984eb..ae5d13fa9 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/fraction.py @@ -14,32 +14,34 @@ import pynini -from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, NEMO_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, NEMO_SIGMA, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.es.graph_utils import ES_MINUS class FractionFst(GraphFst): """ Finite state transducer for classifying fractions - e.g. dos quintos -> fraction { numerator: "2" denominator: "5" } - This class converts fractions with a denominator up to (and including) + e.g. dos quintos -> fraction { numerator: "2" denominator: "5" } + This class converts fractions with a denominator up to (and including) "1/999". - + Fractions with 4 as their denominator, read as "cuarto(s)", are not converted because "room" is also "cuarto", which could cause issues like "quiero reservar un cuarto" -> quiero reservar 1/2". - + Fractions without a numerator are not converted either to prevent issues like: "estaba medio dormido" -> "estaba 1/2 dormido" - + Args: cardinal: CardinalFst ordinal: OrdinalFst + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self, cardinal: GraphFst, ordinal: GraphFst): + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, input_case: str = INPUT_LOWER_CASED): super().__init__(name="fraction", kind="classify") cardinal_graph = cardinal.graph_no_exception @@ -74,7 +76,7 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst): # process negative fractions # e.g. "menos dos tercios" -> "fractions { negative: True numerator: "2" denominator: "3" }" - optional_negative_graph = pynini.closure(pynini.cross("menos", "negative: \"True\"") + NEMO_SPACE, 0, 1) + optional_negative_graph = pynini.closure(pynini.cross(ES_MINUS, "negative: \"True\"") + NEMO_SPACE, 0, 1) # process mixed fractions # e.g. "dos y dos tercios" -> "fractions { integer_part: "2" numerator: "2" denominator: "3" }" @@ -96,7 +98,7 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst): ) proper_fractions_with_medio = optional_negative_graph + proper_fractions_with_medio - self.proper_fractions_with_medio = self.add_tokens(proper_fractions_with_medio) + self.proper_fractions_with_medio = self.add_tokens(proper_fractions_with_medio).optimize() graph = ( optional_negative_graph + optional_integer_part_graph + numerators_graph + NEMO_SPACE + denominators_graph diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/es/taggers/measure.py index 6aea36ede..bdefdcf71 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/measure.py @@ -13,29 +13,35 @@ # limitations under the License. import pynini +from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, NEMO_ALPHA, NEMO_SIGMA, + TO_LOWER, GraphFst, convert_space, delete_extra_space, delete_space, ) -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.es.graph_utils import ES_MINUS class MeasureFst(GraphFst): """ Finite state transducer for classifying measure - e.g. menos doce kilogramos -> measure { cardinal { negative: "true" integer: "12" } units: "kg" } + e.g. menos doce kilogramos -> measure { cardinal { negative: "true" integer: "12" } units: "kg" } Args: cardinal: CardinalFst decimal: DecimalFst + fraction: FractionFst + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst): + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, input_case: str = INPUT_LOWER_CASED): super().__init__(name="measure", kind="classify") cardinal_graph = cardinal.graph_no_exception @@ -45,13 +51,19 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst): math_symbols = pynini.string_file(get_abs_path("data/measures/math_symbols.tsv")) equal_symbol = pynini.string_map([("es igual a", "="), ("igual a", "=")]) + # accept capital letters in units + casing_graph = pynini.closure(TO_LOWER | NEMO_SIGMA).optimize() + graph_unit_singular = pynini.string_file(get_abs_path("data/measures/measurements_singular.tsv")) graph_unit_singular = pynini.invert(graph_unit_singular) # singular -> abbr + graph_unit_singular = pynini.compose(casing_graph, graph_unit_singular).optimize() + graph_unit_plural = pynini.string_file(get_abs_path("data/measures/measurements_plural.tsv")) graph_unit_plural = pynini.invert(graph_unit_plural) # plural -> abbr + graph_unit_plural = pynini.compose(casing_graph, graph_unit_plural).optimize() optional_graph_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("menos", "\"true\"") + delete_extra_space, 0, 1 + pynutil.insert("negative: ") + pynini.cross(ES_MINUS, "\"true\"") + delete_extra_space, 0, 1 ) unit_singular = convert_space(graph_unit_singular) diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/money.py b/nemo_text_processing/inverse_text_normalization/es/taggers/money.py index 357cc8e08..a65d9bcd2 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/money.py @@ -13,17 +13,21 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, NEMO_DIGIT, NEMO_SIGMA, GraphFst, + capitalized_input_graph, convert_space, delete_extra_space, delete_space, insert_space, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): @@ -34,9 +38,10 @@ class MoneyFst(GraphFst): Args: cardinal: CardinalFst decimal: DecimalFst + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self, cardinal: GraphFst, decimal: GraphFst): + def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPUT_LOWER_CASED): super().__init__(name="money", kind="classify") # quantity, integer_part, fractional_part, currency @@ -53,6 +58,22 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): unit_minor_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) unit_minor_plural = pynini.invert(unit_minor_plural) + if input_case == INPUT_CASED: + unit_singular = capitalized_input_graph(unit_singular) + unit_singular_capitalized = pynini.string_file( + get_abs_path("data/money/currency_major_singular_capitalized.tsv") + ) + unit_singular |= pynini.invert(unit_singular_capitalized).optimize() + + unit_plural = capitalized_input_graph(unit_plural) + unit_plural_capitalized = pynini.string_file( + get_abs_path("data/money/currency_major_plural_capitalized.tsv") + ) + unit_plural |= pynini.invert(unit_plural_capitalized).optimize() + + unit_minor_singular = capitalized_input_graph(unit_minor_singular).optimize() + unit_minor_plural = capitalized_input_graph(unit_minor_plural).optimize() + graph_unit_singular = pynutil.insert("currency: \"") + convert_space(unit_singular) + pynutil.insert("\"") graph_unit_plural = pynutil.insert("currency: \"") + convert_space(unit_plural) + pynutil.insert("\"") @@ -65,22 +86,31 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + one_graph = pynini.union("un", "una").optimize() + if input_case == INPUT_CASED: + one_graph |= pynini.union("Un", "Una").optimize() + # twelve dollars (and) fifty cents, zero cents cents_standalone = ( pynutil.insert("morphosyntactic_features: \",\"") # always use a comma in the decimal + insert_space + pynutil.insert("fractional_part: \"") + pynini.union( - pynutil.add_weight(((NEMO_SIGMA - "un") @ cardinal_graph), -0.7) @ add_leading_zero_to_double_digit + pynutil.add_weight(((NEMO_SIGMA - one_graph) @ cardinal_graph), -0.7) + @ add_leading_zero_to_double_digit + delete_space, - pynini.cross("un", "01") + delete_space, + pynini.cross(one_graph, "01") + delete_space, ) + pynutil.insert("\"") ) + and_graph = pynini.union("con", "y").optimize() + if input_case == INPUT_CASED: + and_graph |= pynini.union("Con", "Y").optimize() + optional_cents_standalone = pynini.closure( delete_space - + pynini.closure((pynutil.delete("con") | pynutil.delete('y')) + delete_space, 0, 1) + + pynini.closure(pynutil.delete(and_graph) + delete_space, 0, 1) + insert_space + cents_standalone + pynutil.delete(pynini.union(unit_minor_singular, unit_minor_plural)), @@ -95,7 +125,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): + pynutil.insert("morphosyntactic_features: \",\"") # always use a comma in the decimal + insert_space + pynutil.insert("fractional_part: \"") - + pynini.closure(pynutil.delete("con") + delete_space, 0, 1) + + pynini.closure(pynutil.delete(pynini.union("con", "Con")) + delete_space, 0, 1) + pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7) + pynutil.insert("\""), 0, @@ -104,7 +134,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): graph_integer = ( pynutil.insert("integer_part: \"") - + ((NEMO_SIGMA - "un" - "una") @ cardinal_graph) + + ((NEMO_SIGMA - one_graph) @ cardinal_graph) + pynutil.insert("\"") + delete_extra_space + graph_unit_plural @@ -112,7 +142,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): ) graph_integer |= ( pynutil.insert("integer_part: \"") - + (pynini.cross("un", "1") | pynini.cross("una", "1")) + + pynini.cross(one_graph, "1") + pynutil.insert("\"") + delete_extra_space + graph_unit_singular diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/es/taggers/ordinal.py index 207bbbaf6..7cdcfacc7 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/ordinal.py @@ -13,9 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_SIGMA, + GraphFst, + capitalized_input_graph, + delete_space, +) class OrdinalFst(GraphFst): @@ -24,7 +31,7 @@ class OrdinalFst(GraphFst): vigésimo primero -> ordinal { integer: "21" morphosyntactic_features: "o" } This class converts ordinal up to "millesímo" (one thousandth) exclusive. - Cardinals below ten are not converted (in order to avoid + Cardinals below ten are not converted (in order to avoid e.g. "primero hice ..." -> "1.º hice...", "segunda guerra mundial" -> "2.ª guerra mundial" and any other odd conversions.) @@ -33,9 +40,10 @@ class OrdinalFst(GraphFst): Args: cardinal: CardinalFst + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self, cardinal: GraphFst): + def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): super().__init__(name="ordinal", kind="classify") cardinal_graph = cardinal.graph_no_exception @@ -45,9 +53,22 @@ def __init__(self, cardinal: GraphFst): graph_ties = pynini.string_file(get_abs_path("data/ordinals/ties.tsv")) graph_hundreds = pynini.string_file(get_abs_path("data/ordinals/hundreds.tsv")) + if input_case == INPUT_CASED: + graph_digit |= pynini.string_file(get_abs_path("data/ordinals/digit_capitalized.tsv")).optimize() + graph_teens |= pynini.string_file(get_abs_path("data/ordinals/teen_capitalized.tsv")).optimize() + graph_twenties |= pynini.string_file(get_abs_path("data/ordinals/twenties_capitalized.tsv")).optimize() + graph_ties |= pynini.string_file(get_abs_path("data/ordinals/ties_capitalized.tsv")).optimize() + graph_hundreds |= pynini.string_file(get_abs_path("data/ordinals/hundreds_capitalized.tsv")).optimize() + full_graph_ties = graph_ties | (graph_ties + pynini.cross(" ", "y") + graph_digit) - ordinal_graph_union = pynini.union(graph_digit, graph_teens, graph_twenties, full_graph_ties, graph_hundreds,) + ordinal_graph_union = pynini.union( + graph_digit, + graph_teens, + graph_twenties, + full_graph_ties, + graph_hundreds, + ) accept_o_endings = NEMO_SIGMA + pynini.accep("o") accept_a_endings = NEMO_SIGMA + pynini.accep("a") @@ -64,7 +85,7 @@ def __init__(self, cardinal: GraphFst): graph_a_suffix = (optional_numbers_in_front + ordinal_graph_a) @ cardinal_graph graph_er_suffix = (optional_numbers_in_front + ordinal_graph_er) @ cardinal_graph - self.graph_masc_num_no_exception = graph_o_suffix + self.graph_masc_num_no_exception = graph_o_suffix.optimize() # don't convert ordinals from one to nine inclusive graph_exception = pynini.project(pynini.union(graph_digit), 'input') @@ -72,6 +93,11 @@ def __init__(self, cardinal: GraphFst): graph_a_suffix = (pynini.project(graph_a_suffix, "input") - graph_exception.arcsort()) @ graph_a_suffix graph_er_suffix = (pynini.project(graph_er_suffix, "input") - graph_exception.arcsort()) @ graph_er_suffix + if input_case == INPUT_CASED: + graph_o_suffix = capitalized_input_graph(graph_o_suffix) + graph_a_suffix = capitalized_input_graph(graph_a_suffix) + graph_er_suffix = capitalized_input_graph(graph_er_suffix) + graph = ( pynutil.insert("integer: \"") + graph_o_suffix diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/es/taggers/punctuation.py index 12405d5c5..c34f732ed 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/punctuation.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/punctuation.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class PunctuationFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/es/taggers/telephone.py index 0f6b5f003..8c73ca434 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/telephone.py @@ -13,14 +13,21 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + GraphFst, + capitalized_input_graph, + delete_space, +) +from nemo_text_processing.text_normalization.es.graph_utils import ES_PLUS class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone numbers, e.g. + Finite state transducer for classifying telephone numbers, e.g. uno dos tres uno dos tres cinco seis siete ocho -> { number_part: "123-123-5678" }. If 10 digits are spoken, they are grouped as 3+3+4 (eg. 123-456-7890). If 9 digits are spoken, they are grouped as 3+3+3 (eg. 123-456-789). @@ -30,9 +37,12 @@ class TelephoneFst(GraphFst): "twelve thirty four" = "1234". (we ignore more complicated cases such as "three hundred and two" or "three nines"). + + Args: + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self): + def __init__(self, input_case: str = INPUT_LOWER_CASED): super().__init__(name="telephone", kind="classify") # create `single_digits` and `double_digits` graphs as these will be @@ -41,8 +51,16 @@ def __init__(self): graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv")) + graph_zero = pynini.cross("cero", "0") + + if input_case == INPUT_CASED: + graph_digit = capitalized_input_graph(graph_digit) + graph_ties = capitalized_input_graph(graph_ties) + graph_teen = capitalized_input_graph(graph_teen) + graph_twenties = capitalized_input_graph(graph_twenties) + graph_zero = pynini.cross(pynini.union("cero", "Cero"), "0").optimize() - single_digits = graph_digit.optimize() | pynini.cross("cero", "0") + single_digits = graph_digit.optimize() | graph_zero double_digits = pynini.union( graph_twenties, @@ -57,7 +75,7 @@ def __init__(self): digit_thrice = digit_twice + pynutil.delete(" ") + single_digits # accept `doble cero` -> `00` and `triple ocho` -> `888` - digit_words = pynini.union(graph_digit.optimize(), pynini.cross("cero", "0")).invert() + digit_words = pynini.union(graph_digit.optimize(), graph_zero).invert() doubled_digit = pynini.union( *[ @@ -92,7 +110,10 @@ def __init__(self): # Denormalized phone numbers are grouped in sets of 3 or 4 digits group_of_two = pynini.union(doubled_digit, digit_twice, double_digits) - group_of_three = pynini.union(tripled_digit, single_digits + pynutil.delete(" ") + group_of_two,) + group_of_three = pynini.union( + tripled_digit, + single_digits + pynutil.delete(" ") + group_of_two, + ) group_of_four = pynini.union( group_of_two + pynutil.delete(" ") + group_of_two, @@ -111,14 +132,26 @@ def __init__(self): # 8-digit option eight_digit_graph = group_of_four + insert_separator + group_of_four + plus = pynini.accep("más") + if input_case == INPUT_CASED: + plus |= ES_PLUS + # optionally denormalize country codes optional_country_code = pynini.closure( - pynini.cross("más ", "+") + (single_digits | group_of_two | group_of_three) + insert_separator, 0, 1 + pynini.cross(plus, "+") + + delete_space + + (single_digits | group_of_two | group_of_three) + + insert_separator, + 0, + 1, ) + ext_phrase = pynini.accep(" extensión ") + if input_case == INPUT_CASED: + ext_phrase = pynini.union(" extensión ", " Extensión ") # optionally denormalize extensions optional_extension = pynini.closure( - pynini.cross(" extensión ", " ext. ") + (single_digits | group_of_two | group_of_three), 0, 1 + pynini.cross(ext_phrase, " ext. ") + (single_digits | group_of_two | group_of_three), 0, 1 ) number_part = ( @@ -130,5 +163,8 @@ def __init__(self): number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") graph = number_part + if input_case == INPUT_CASED: + graph |= capitalized_input_graph(graph) + final_graph = self.add_tokens(graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/time.py b/nemo_text_processing/inverse_text_normalization/es/taggers/time.py index 6e57b5a77..f33c7c1b1 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/time.py @@ -14,15 +14,19 @@ import pynini +from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, GraphFst, + capitalized_input_graph, convert_space, delete_extra_space, delete_space, insert_space, ) -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.es.graph_utils import ES_MINUS, ES_PLUS class TimeFst(GraphFst): @@ -41,27 +45,30 @@ class TimeFst(GraphFst): e.g. cuarto para las dos -> time { minutes: "45" hours: "la 1" } Note that times on the hour (e.g. "las dos" i.e. "two o'clock") do not get - converted into a time format. This is to avoid converting phrases that are + converted into a time format. This is to avoid converting phrases that are not part of a time phrase (e.g. "las dos personas" i.e. "the two people") e.g. las dos -> tokens { name: "las" } tokens { name: "dos" } - However, if a time on the hour is followed by a suffix (indicating 'a.m.' + However, if a time on the hour is followed by a suffix (indicating 'a.m.' or 'p.m.'), it will be converted. e.g. las dos pe eme -> time { hours: "las 2" minutes: "00" suffix: "p.m." } - - In the same way, times without a preceding article are not converted. This is + + In the same way, times without a preceding article are not converted. This is to avoid converting ranges or complex fractions e.g. dos y media -> tokens { name: "dos" } tokens { name: "y" } tokens { name: "media" } - However, if a time without an article is followed by a suffix (indicating 'a.m.' + However, if a time without an article is followed by a suffix (indicating 'a.m.' or 'p.m.'), it will be converted. e.g. dos y media p m -> time { hours: "2" minutes: "30" suffix: "p.m." } - Note that although the TimeFst verbalizer can accept 'zone' (timezone) fields, + Note that although the TimeFst verbalizer can accept 'zone' (timezone) fields, so far the rules have not been added to the TimeFst tagger to process timezones (to keep the rules simple, and because timezones are not very often specified in Spanish.) + + Args: + input_case: accepting either "lower_cased" or "cased" input. """ - def __init__(self): + def __init__(self, input_case: str = INPUT_LOWER_CASED): super().__init__(name="time", kind="classify") suffix_graph = pynini.string_file(get_abs_path("data/time/time_suffix.tsv")) @@ -81,7 +88,7 @@ def __init__(self): graph_teen, (graph_ties + pynutil.insert("0")), (graph_ties + pynutil.delete(" y ") + graph_digit), - ) + ).optimize() # note that graph_hour will start from 2 hours # "1 o'clock" will be treated differently because it @@ -89,24 +96,51 @@ def __init__(self): digits_2_to_23 = [str(digits) for digits in range(2, 24)] digits_1_to_59 = [str(digits) for digits in range(1, 60)] - graph_1oclock = pynini.cross("la una", "la 1") - graph_hour = pynini.cross("las ", "las ") + graph_1_to_100 @ pynini.union(*digits_2_to_23) + one_o_clock = pynini.accep("la una") + article = pynini.accep("las ") + half = pynini.accep("media") + quarter = pynini.accep("cuarto") + and_graph = pynini.union("y", "con") + hours_word_graph = pynini.accep(" horas") + minutes_word_graph = pynini.union(" minuto", " minutos") + + if input_case == INPUT_CASED: + suffix_graph |= pynini.string_file(get_abs_path("data/time/time_suffix_cased.tsv")).optimize() + time_zones |= pynini.invert(pynini.string_file(get_abs_path("data/time/time_zone_cased.tsv"))).optimize() + graph_digit |= capitalized_input_graph(graph_digit).optimize() + graph_1_to_100 |= capitalized_input_graph(graph_1_to_100).optimize() + article |= pynini.accep("Las ").optimize() + half |= pynini.accep("Media").optimize() + quarter |= pynini.accep("Cuarto").optimize() + and_graph |= pynini.union("Y", "Con").optimize() + hours_word_graph |= pynini.accep(" Horas").optimize() + minutes_word_graph |= pynini.union(" Minuto", " Minutos").optimize() + + graph_one_o_clock = pynini.cross(one_o_clock, "la 1") + if input_case == INPUT_CASED: + graph_one_o_clock |= pynini.cross(pynini.accep("la Una"), "la 1") + one_o_clock_capitalized = pynini.union("La Una", "La una") + graph_one_o_clock |= pynini.cross(one_o_clock_capitalized, "La 1").optimize() + + graph_hour = article + graph_1_to_100 @ pynini.union(*digits_2_to_23) graph_minute = graph_1_to_100 @ pynini.union(*digits_1_to_59) - graph_minute_verbose = pynini.cross("media", "30") | pynini.cross("cuarto", "15") + graph_minute_verbose = pynini.cross(half, "30") | pynini.cross(quarter, "15") - final_graph_hour = pynutil.insert("hours: \"") + (graph_1oclock | graph_hour) + pynutil.insert("\"") + final_graph_hour = pynutil.insert("hours: \"") + (graph_one_o_clock | graph_hour) + pynutil.insert("\"") final_graph_minute = ( pynutil.insert("minutes: \"") - + pynini.closure((pynutil.delete("y") | pynutil.delete("con")) + delete_space, 0, 1) + + pynini.closure(pynutil.delete(and_graph) + delete_space, 0, 1) + (graph_minute | graph_minute_verbose) + + pynini.closure(pynutil.delete(minutes_word_graph), 0, 1) + + pynini.closure(pynutil.delete(hours_word_graph), 0, 1) + pynutil.insert("\"") - ) + ).optimize() # g m t más tres -> las 2:00 p.m. gmt+3 digits_1_to_23 = [str(digits) for digits in range(1, 24)] offset = graph_1_to_100 @ pynini.union(*digits_1_to_23) - sign = pynini.cross("más", "+") | pynini.cross("menos", "-") + sign = pynini.cross(ES_PLUS, "+") | pynini.cross(ES_MINUS, "-") full_offset = pynutil.delete(" ") + sign + pynutil.delete(" ") + offset graph_offset = pynini.closure(full_offset, 0, 1) graph_time_zones = pynini.accep(" ") + time_zones + graph_offset @@ -126,12 +160,14 @@ def __init__(self): ) # las nueve a eme (only convert on-the-hour times if they are followed by a suffix) - graph_1oclock_with_suffix = pynini.closure(pynini.accep("la "), 0, 1) + pynini.cross("una", "1") - graph_hour_with_suffix = pynini.closure(pynini.accep("las "), 0, 1) + graph_1_to_100 @ pynini.union( - *digits_2_to_23 + graph_one_o_clock_with_suffix = pynini.closure(pynini.union("la ", "La "), 0, 1) + pynini.cross( + pynini.union("una", "Una"), "1" ) + graph_hour_with_suffix = pynini.closure(article, 0, 1) + graph_1_to_100 @ pynini.union(*digits_2_to_23) final_graph_hour_with_suffix = ( - pynutil.insert("hours: \"") + (graph_1oclock_with_suffix | graph_hour_with_suffix) + pynutil.insert("\"") + pynutil.insert("hours: \"") + + (graph_one_o_clock_with_suffix | graph_hour_with_suffix) + + pynutil.insert("\"") ) graph_hsuffix = ( @@ -170,14 +206,14 @@ def __init__(self): + delete_extra_space + pynutil.insert("minutes: \"") + delete_space - + pynutil.delete("menos") + + pynutil.delete(ES_MINUS) + delete_space + pynini.union( - pynini.cross("cinco", "55"), - pynini.cross("diez", "50"), - pynini.cross("cuarto", "45"), - pynini.cross("veinte", "40"), - pynini.cross("veinticinco", "30"), + pynini.cross(pynini.union("cinco", "Cinco"), "55"), + pynini.cross(pynini.union("diez", "Diez"), "50"), + pynini.cross(pynini.union("cuarto", "Cuarto"), "45"), + pynini.cross(pynini.union("veinte", "Veinte"), "40"), + pynini.cross(pynini.union("veinticinco", "Veinticinco"), "35"), ) + pynutil.insert("\"") ) diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py index 01b77f429..26483fb83 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.es.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.es.taggers.date import DateFst from nemo_text_processing.inverse_text_normalization.es.taggers.decimal import DecimalFst @@ -36,7 +37,7 @@ delete_space, generator_main, ) -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -67,35 +68,37 @@ def __init__( far_file = os.path.join(cache_dir, f"es_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") - cardinal = CardinalFst() + cardinal = CardinalFst(input_case=input_case) cardinal_graph = cardinal.fst - ordinal = OrdinalFst(cardinal) + ordinal = OrdinalFst(cardinal, input_case=input_case) ordinal_graph = ordinal.fst - decimal = DecimalFst(cardinal) + decimal = DecimalFst(cardinal, input_case=input_case) decimal_graph = decimal.fst - fraction = FractionFst(cardinal, ordinal) + fraction = FractionFst(cardinal, ordinal, input_case=input_case) fraction_graph = fraction.fst - measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction).fst - date_graph = DateFst(cardinal).fst + measure_graph = MeasureFst( + cardinal=cardinal, decimal=decimal, fraction=fraction, input_case=input_case + ).fst + date_graph = DateFst(cardinal, input_case=input_case).fst word_graph = WordFst().fst - time_graph = TimeFst().fst - money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst - whitelist_graph = WhiteListFst(input_file=whitelist).fst + time_graph = TimeFst(input_case=input_case).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, input_case=input_case).fst + whitelist_graph = WhiteListFst(input_file=whitelist, input_case=input_case).fst punct_graph = PunctuationFst().fst - electronic_graph = ElectronicFst().fst - telephone_graph = TelephoneFst().fst + electronic_graph = ElectronicFst(input_case=input_case).fst + telephone_graph = TelephoneFst(input_case=input_case).fst classify = ( pynutil.add_weight(whitelist_graph, 1.01) - | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(time_graph, 1.08) | pynutil.add_weight(date_graph, 1.09) | pynutil.add_weight(decimal_graph, 1.09) | pynutil.add_weight(fraction_graph, 1.09) @@ -104,7 +107,7 @@ def __init__( | pynutil.add_weight(ordinal_graph, 1.6) | pynutil.add_weight(money_graph, 1.6) | pynutil.add_weight(telephone_graph, 1.6) - | pynutil.add_weight(electronic_graph, 1.6) + | pynutil.add_weight(electronic_graph, 2.96) | pynutil.add_weight(word_graph, 100) ) @@ -121,4 +124,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/es/taggers/whitelist.py index 2d31cede5..55f10efc6 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/whitelist.py @@ -12,11 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + import pynini -from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels + class WhiteListFst(GraphFst): """ @@ -27,16 +36,50 @@ class WhiteListFst(GraphFst): Whitelisted tokens are defined and loaded from "data/whitelist.tsv" (unless input_file specified). Args: + input_case: accepting either "lower_cased" or "cased" input. input_file: path to a file with whitelist replacements (each line of the file: written_form\tspoken_form\n), e.g. nemo_text_processing/inverse_text_normalization/es/data/whitelist.tsv """ - def __init__(self, input_file: str = None): + def __init__(self, input_case: str = INPUT_LOWER_CASED, input_file: str = None): super().__init__(name="whitelist", kind="classify") - if input_file: - whitelist = pynini.string_file(input_file).invert() - else: - whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert() + def get_whitelist_graph(input_file: str): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" Y ", " y "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + if input_file is None: + input_file = get_abs_path("data/whitelist.tsv") + + if not os.path.exists(input_file): + raise ValueError(f"Whitelist file {input_file} not found") + + whitelist = get_whitelist_graph(input_file) + graph = pynutil.insert("name: \"") + convert_space(whitelist) + pynutil.insert("\"") self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/taggers/word.py b/nemo_text_processing/inverse_text_normalization/es/taggers/word.py index 57f143d0d..b09f941e2 100644 --- a/nemo_text_processing/inverse_text_normalization/es/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/es/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/utils.py b/nemo_text_processing/inverse_text_normalization/es/utils.py index f6e06f793..bedda6391 100644 --- a/nemo_text_processing/inverse_text_normalization/es/utils.py +++ b/nemo_text_processing/inverse_text_normalization/es/utils.py @@ -21,7 +21,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py index e6737be6e..46fca2a1c 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/date.py index 234fdc296..61b9e0b84 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/date.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.es.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -20,7 +23,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DateFst(GraphFst): @@ -31,6 +33,9 @@ class DateFst(GraphFst): def __init__(self): super().__init__(name="date", kind="verbalize") + graph_month = pynini.string_file(get_abs_path("data/dates/months.tsv")) + graph_month |= pynini.string_file(get_abs_path("data/dates/months_cased.tsv")) + year = ( pynutil.delete("year:") + delete_space @@ -38,13 +43,7 @@ def __init__(self): + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") ) - month = ( - pynutil.delete("month:") - + delete_space - + pynutil.delete("\"") - + pynini.closure(NEMO_NOT_QUOTE, 1) - + pynutil.delete("\"") - ) + month = pynutil.delete("month:") + delete_space + pynutil.delete("\"") + graph_month + pynutil.delete("\"") day = ( pynutil.delete("day:") + delete_space diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/decimal.py index b443733a2..c9b060ec1 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/electronic.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/electronic.py index cf6bdc779..069716123 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class ElectronicFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/fraction.py index eae72ab4b..e553d95e9 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/fraction.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/measure.py index 0bd8f7460..6162f0c20 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/measure.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, GraphFst, delete_extra_space, delete_space, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/money.py index a52371f1e..60e9b7aeb 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py index 8cfb15095..c1c9bdb46 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/telephone.py index bc32f62fe..8364c250b 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/telephone.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class TelephoneFst(GraphFst): """ Finite state transducer for verbalizing telephone, e.g. telephone { number_part: "123-123-5678" } - -> 123-123-5678 + -> 123-123-5678 """ def __init__(self): diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/time.py index 15b7ddbf1..44892b0be 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -34,12 +35,14 @@ def __init__(self): super().__init__(name="time", kind="verbalize") add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) - # hour may or may not include preposition ("la" or "las") + # hour may or may not include article ("la" or "las") + article = pynini.union("la ", "las ", "La ", "Las ") + hour = ( pynutil.delete("hours:") + delete_space + pynutil.delete("\"") - + pynini.closure(pynini.union("la ", "las "), 0, 1) + + pynini.closure(article, 0, 1) + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") ) diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py index 26d09996b..5c45ff66f 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/verbalize_final.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.es.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.es.verbalizers.word import WordFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now """ diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py index cc231a46b..d54b8509a 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WhiteListFst(GraphFst): """ @@ -33,5 +34,5 @@ def __init__(self): + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete("\"") ) - graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/es/verbalizers/word.py index 3a5ba96b9..197155d92 100644 --- a/nemo_text_processing/inverse_text_normalization/es/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/es/verbalizers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WordFst(GraphFst): """ @@ -27,6 +28,6 @@ def __init__(self): super().__init__(name="word", kind="verbalize") chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/es_en/data/es_whitelist.tsv b/nemo_text_processing/inverse_text_normalization/es_en/data/es_whitelist.tsv index 60253820a..cfa739c1a 100644 --- a/nemo_text_processing/inverse_text_normalization/es_en/data/es_whitelist.tsv +++ b/nemo_text_processing/inverse_text_normalization/es_en/data/es_whitelist.tsv @@ -1,16 +1,16 @@ -ud. usted -uds. ustedes -vd. vosotros -vds. vosotros -dr. doctor -dra. doctora -d. don -da. doña -ee. uu. estados unidos +Ud. usted +Uds. ustedes +Vd. vosotros +Vds. vosotros +Dr. doctor +Dra. doctora +D. don +Da. doña +EE. UU. estados unidos p.ej. por ejemplo -prof. profesor -profa. profesora -sr. señor -sra. señora -srta. señorita -etc. etcétera +Prof. profesor +Profa. profesora +Sr. señor +Sra. señora +Srta. señorita +etc. etcétera \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/es_en/graph_utils.py b/nemo_text_processing/inverse_text_normalization/es_en/graph_utils.py index 164ebcdbb..0396b5b4c 100644 --- a/nemo_text_processing/inverse_text_normalization/es_en/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/es_en/graph_utils.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.es.utils import get_abs_path from pynini.lib import pynutil +from nemo_text_processing.text_normalization.es.utils import get_abs_path + def int_to_roman(fst: 'pynini.FstLike') -> 'pynini.FstLike': """ diff --git a/nemo_text_processing/inverse_text_normalization/es_en/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/es_en/taggers/tokenize_and_classify.py index c5b948934..ebef11007 100644 --- a/nemo_text_processing/inverse_text_normalization/es_en/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/es_en/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst as EnCardinalFst from nemo_text_processing.inverse_text_normalization.en.taggers.date import DateFst as EnDateFst from nemo_text_processing.inverse_text_normalization.en.taggers.decimal import DecimalFst as EnDecimalFst @@ -49,7 +50,7 @@ delete_space, generator_main, ) -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -85,31 +86,33 @@ def __init__( far_file = os.path.join(cache_dir, f"es_en_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") - cardinal = CardinalFst() + cardinal = CardinalFst(input_case=input_case) cardinal_graph = cardinal.fst - ordinal = OrdinalFst(cardinal) + ordinal = OrdinalFst(cardinal, input_case=input_case) ordinal_graph = ordinal.fst - decimal = DecimalFst(cardinal) + decimal = DecimalFst(cardinal, input_case=input_case) decimal_graph = decimal.fst - fraction = FractionFst(cardinal, ordinal) + fraction = FractionFst(cardinal, ordinal, input_case=input_case) fraction_graph = fraction.fst - measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction).fst - date_graph = DateFst(cardinal).fst + measure_graph = MeasureFst( + cardinal=cardinal, decimal=decimal, fraction=fraction, input_case=input_case + ).fst + date_graph = DateFst(cardinal, input_case=input_case).fst word_graph = WordFst().fst - time_graph = TimeFst().fst - money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst + time_graph = TimeFst(input_case=input_case).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, input_case=input_case).fst whitelist_graph = WhiteListFst(input_file=whitelist).fst punct_graph = PunctuationFst().fst - electronic_graph = ElectronicFst().fst - telephone_graph = TelephoneFst().fst + electronic_graph = ElectronicFst(input_case=input_case).fst + telephone_graph = TelephoneFst(input_case=input_case).fst en_cardinal = EnCardinalFst(input_case=input_case) en_cardinal_graph = en_cardinal.fst @@ -150,7 +153,7 @@ def __init__( | pynutil.add_weight(en_money_graph, 1.1) | pynutil.add_weight(telephone_graph, 1.6) | pynutil.add_weight(en_telephone_graph, 1.1) - | pynutil.add_weight(electronic_graph, 1.6) + | pynutil.add_weight(electronic_graph, 2.3) | pynutil.add_weight(en_electronic_graph, 1.1) | pynutil.add_weight(word_graph, 100) | pynutil.add_weight(en_word_graph, 120) @@ -174,4 +177,4 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/es_en/utils.py b/nemo_text_processing/inverse_text_normalization/es_en/utils.py index f6e06f793..bedda6391 100644 --- a/nemo_text_processing/inverse_text_normalization/es_en/utils.py +++ b/nemo_text_processing/inverse_text_normalization/es_en/utils.py @@ -21,7 +21,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize.py index 8e540a616..483d083c8 100644 --- a/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.verbalizers.cardinal import CardinalFst as EnCardinalFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.date import DateFst as EnDateFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.decimal import DecimalFst as EnDecimalFst @@ -34,7 +36,6 @@ from nemo_text_processing.inverse_text_normalization.es.verbalizers.time import TimeFst from nemo_text_processing.inverse_text_normalization.es.verbalizers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst -from pynini.lib import pynutil class VerbalizeFst(GraphFst): @@ -74,10 +75,12 @@ def __init__(self): en_date_graph = EnDateFst().fst en_whitelist_graph = EnWhiteListFst().fst en_telephone_graph = EnTelephoneFst().fst + en_time_graph = EnTimeFst().fst en_electronic_graph = EnElectronicFst().fst graph = ( - time_graph + en_time_graph + | pynutil.add_weight(time_graph, 1.1) | date_graph | pynutil.add_weight(en_date_graph, 1.1) | money_graph diff --git a/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize_final.py index 65d9b91c4..e46b6db56 100644 --- a/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/es_en/verbalizers/verbalize_final.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.es.verbalizers.word import WordFst from nemo_text_processing.inverse_text_normalization.es_en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now """ diff --git a/nemo_text_processing/inverse_text_normalization/fr/graph_utils.py b/nemo_text_processing/inverse_text_normalization/fr/graph_utils.py index 9edfed206..676574c79 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/fr/graph_utils.py @@ -13,19 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import string from pathlib import Path from typing import Dict import pynini -from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path from pynini import Far from pynini.examples import plurals from pynini.export import export from pynini.lib import byte, pynutil, utf8 +from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path +from nemo_text_processing.utils.logging import logger + NEMO_CHAR = utf8.VALID_UTF8_CHAR NEMO_DIGIT = byte.DIGIT @@ -34,9 +35,9 @@ NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() NEMO_HEX = pynini.union(*string.hexdigits).optimize() -NEMO_NON_BREAKING_SPACE = u"\u00A0" +NEMO_NON_BREAKING_SPACE = u"\u00a0" NEMO_SPACE = " " -NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00A0").optimize() +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() @@ -80,7 +81,7 @@ def generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]): for rule, graph in graphs.items(): exporter[rule] = graph.optimize() exporter.close() - logging.info(f'Created {file_name}') + logger.info(f'Created {file_name}') def get_plurals(fst): @@ -187,4 +188,4 @@ def delete_tokens(self, fst) -> 'pynini.FstLike': + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/cardinal.py index 2c8fd6c0c..ea1fcf8ea 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/cardinal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -23,7 +25,6 @@ delete_hyphen, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil def rewrite(cardinal: 'pynini.FstLike') -> 'pynini.FstLike': @@ -34,9 +35,9 @@ def rewrite(cardinal: 'pynini.FstLike') -> 'pynini.FstLike': In cases where original orthography is current, or string is mixture of two orthographies, will render invalid form that will not pass through CardinalFst e.g. deux-mille cent-vingt-trois -> "deux##vingt-trois" ('#' is not accepted in cardinal FST and will fail to convert.) - e.g. deux + e.g. deux - Args: + Args: cardinal: cardinal FST """ @@ -89,13 +90,13 @@ def rewrite(cardinal: 'pynini.FstLike') -> 'pynini.FstLike': class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals - e.g. mois vingt-trois -> cardinal { negative: "-" integer: "23"} + e.g. mois vingt-trois -> cardinal { negative: "-" integer: "23"} This class converts cardinals up to (but not including) "un-quatrillion", i.e up to "one septillion" in English (10^{24}). - Cardinals below nine are not converted (in order to avoid + Cardinals below nine are not converted (in order to avoid "j'ai un pomme." --> "j'ai 1 pomme" and any other odd conversions.) This transducer accomodates both traditional hyphenation of numbers ('-' for most numbers <100) - and current hyphenation (all elements of number are hyphenated), prioritizing the latter. + and current hyphenation (all elements of number are hyphenated), prioritizing the latter. e.g cent cinquante et un -> cardinal { integer: "151"} cent-cinquante-et-un -> cardinal { integer: "151"} This is done through a context dependent rewrite that attempts to map old spelling to new. @@ -247,18 +248,13 @@ def __init__(self): self.graph_no_exception = graph.optimize() # save self.numbers_up_to_thousand for use in DecimalFst - digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT ** 2) | (NEMO_DIGIT ** 3) + digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3) numbers_up_to_thousand = pynini.compose(graph, digits_up_to_thousand).optimize() self.numbers_up_to_thousand = numbers_up_to_thousand # save self.numbers_up_to_million for use in DecimalFst digits_up_to_million = ( - NEMO_DIGIT - | (NEMO_DIGIT ** 2) - | (NEMO_DIGIT ** 3) - | (NEMO_DIGIT ** 4) - | (NEMO_DIGIT ** 5) - | (NEMO_DIGIT ** 6) + NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3) | (NEMO_DIGIT**4) | (NEMO_DIGIT**5) | (NEMO_DIGIT**6) ) numbers_up_to_million = pynini.compose(graph, digits_up_to_million).optimize() self.numbers_up_to_million = numbers_up_to_million diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/date.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/date.py index 1412111b4..68d35741c 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, delete_extra_space from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class DateFst(GraphFst): @@ -45,7 +46,9 @@ def __init__(self, cardinal: GraphFst): day_graph = self.cardinal | pynini.cross("premier", "1") # Premier is only ordinal used for dates day_graph = pynutil.insert("day: \"") + day_graph + pynutil.insert("\"") optional_graph_year = pynini.closure( - delete_extra_space + pynutil.insert("year: \"") + year_graph + pynutil.insert("\""), 0, 1, + delete_extra_space + pynutil.insert("year: \"") + year_graph + pynutil.insert("\""), + 0, + 1, ) graph_dmy = day_graph + delete_extra_space + month_graph + optional_graph_year diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/decimal.py index 035d70fab..9f6341cf4 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_DIGIT, GraphFst, @@ -21,7 +23,6 @@ delete_space, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_thousand: 'pynini.FstLike') -> 'pynini.FstLike': @@ -30,9 +31,9 @@ def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_thousand: 'pynini.Fst e.g. one million -> integer_part: "1" quantity: "million" e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million" - Will tag cases up to denominations of tens of hundreds of thousand. 'douze cent mille millions' -> 1 200 000 millions + Will tag cases up to denominations of tens of hundreds of thousand. 'douze cent mille millions' -> 1 200 000 millions - Args: + Args: decimal: decimal FST cardinal_up_to_million: cardinal FST """ @@ -78,7 +79,7 @@ class DecimalFst(GraphFst): This decimal rule assumes that decimals can be pronounced as: (a cardinal) + ('virgule') plus (any sequence of cardinals <1 million, including 'zero') - Also writes large numbers in shortened form, e.g. + Also writes large numbers in shortened form, e.g. e.g. un virgule deux-six-million -> decimal { negative: "false" integer_part: "1" fractional_part: "26" quantity: "million" } e.g. deux-million -> decimal { negative: "false" integer_part: "2" quantity: "millions" } e.g. moins cent-vingt-quatre-millions -> decimal { negative: "true" integer_part: "124" quantity: "millions" } diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/electronic.py index 15d826650..36e54fad4 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_ALPHA, GraphFst, insert_space from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class ElectronicFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/fraction.py index e05323724..94b87bfd5 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/fraction.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_CHAR, GraphFst, @@ -20,7 +22,6 @@ delete_space, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class FractionFst(GraphFst): @@ -29,7 +30,7 @@ class FractionFst(GraphFst): e.g. demi -> tokens { fraction { numerator: "1" denominator: "2" } } e.g. un et demi -> tokens { fraction { integer_part: "1" numerator: "1" denominator: "2" } } e.g. trois et deux centième -> tokens { fraction { integer_part: "3" numerator: "2" denominator: "100" } } - + Args: cardinal: OrdinalFst """ diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/measure.py index d9636286c..0bd3298c3 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( GraphFst, delete_extra_space, @@ -20,7 +22,6 @@ get_singulars, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class MeasureFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/money.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/money.py index 17f473aa2..94c0dd2eb 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_DIGIT, GraphFst, @@ -20,7 +22,6 @@ delete_space, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/ordinal.py index 29da19b85..629fc0e26 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_SIGMA, GraphFst, delete_space from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): @@ -23,7 +24,7 @@ class OrdinalFst(GraphFst): Finite state transducer for classifying ordinal vingt-deuxième -> ordinal { integer: "22" morphosyntactic_features: "e" } - Also notes specific nouns that have unique normalization conventions. + Also notes specific nouns that have unique normalization conventions. For instance, 'siècles' are rendered in roman numerals when given an ordinal adjective. e.g. dix-neuvième siècle -> XIXe diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/punctuation.py index 458497aa6..da26ba825 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/punctuation.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/punctuation.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst + class PunctuationFst(GraphFst): """ @@ -27,7 +28,7 @@ def __init__(self): super().__init__(name="punctuation", kind="classify") s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~" - guillemets = "\u00AB" + "\u00BB" # quotation marks in French. + guillemets = "\u00ab" + "\u00bb" # quotation marks in French. s += guillemets punct = pynini.union(*s) diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/telephone.py index b81cc0dfd..c532cfd06 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/telephone.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( GraphFst, delete_hyphen, @@ -20,13 +22,12 @@ insert_space, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class TelephoneFst(GraphFst): """ Finite state transducer for classifying telephone numbers. Assumes conventional grouping for Metropolitan France (and overseas departments) - (two number sequences are grouped as individual cardinals) or digit by digit (chiffre-par-chiffre) e.g. + (two number sequences are grouped as individual cardinals) or digit by digit (chiffre-par-chiffre) e.g. "zero un quatre-vingt-deux zero deux vingt-deux cinquante" -> { number_part: "01 42 02 22 50" } "zero un quatre deux zero deux deux deux cinq zero" -> { number_part: "01 42 02 22 50" } diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/time.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/time.py index 499701a1e..5113d50e8 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/time.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, delete_space from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py index a4cea1e7b..88774456e 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( GraphFst, delete_extra_space, @@ -36,7 +37,7 @@ from nemo_text_processing.inverse_text_normalization.fr.taggers.whitelist import WhiteListFst from nemo_text_processing.inverse_text_normalization.fr.taggers.word import WordFst from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -67,9 +68,9 @@ def __init__( far_file = os.path.join(cache_dir, f"fr_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") cardinal = CardinalFst() cardinal_graph = cardinal.fst @@ -121,4 +122,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/whitelist.py index 363e64bfa..722dc84b4 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, convert_space from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/taggers/word.py b/nemo_text_processing/inverse_text_normalization/fr/taggers/word.py index 95d05ce6a..21b852910 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/fr/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/fr/utils.py b/nemo_text_processing/inverse_text_normalization/fr/utils.py index f6e06f793..bedda6391 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/utils.py +++ b/nemo_text_processing/inverse_text_normalization/fr/utils.py @@ -21,7 +21,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py index 4aca2bac2..bc4e76fa2 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/date.py index 47304c1e1..62304b0d2 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/date.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_space, ) -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py index dc00842cd..3e654b859 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_DIGIT, NEMO_NON_BREAKING_SPACE, @@ -20,13 +22,12 @@ GraphFst, delete_space, ) -from pynini.lib import pynutil class NumberParser(GraphFst): """ - Finite state transducer for parsing strings of digis. Breaks up digit strings into groups of three for - strings of digits of four or more (inclusive). Groupings are separated by non-breaking space. + Finite state transducer for parsing strings of digis. Breaks up digit strings into groups of three for + strings of digits of four or more (inclusive). Groupings are separated by non-breaking space. e.g. '1000' -> '1 000' e.g. '1000,33333' -> '1 000,333 33 """ @@ -45,7 +46,7 @@ def __init__(self): super().__init__(name="decimal", kind="verbalize") # Need parser to group digits by threes - exactly_three_digits = NEMO_DIGIT ** 3 + exactly_three_digits = NEMO_DIGIT**3 at_most_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) space_every_three_integer = ( diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py index e4cc75cbd..72113be63 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class ElectronicFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py index dce25d949..f7d163f23 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/fraction.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space, ) -from pynini.lib import pynutil class FractionFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/measure.py index ad0f2f98c..8b49c0eb1 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/money.py index db4725684..15dc72c94 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/money.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_space, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py index 96d8c2403..3179af643 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/ordinal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, @@ -20,7 +22,6 @@ delete_space, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): @@ -60,12 +61,12 @@ def __init__(self): graph_roman_hundreds = pynini.string_file(get_abs_path("data/roman/hundreds_large.tsv")).invert() graph_roman_zero_digit = pynutil.delete("0") - graph_roman_hundreds = NEMO_DIGIT ** 3 @ ( + graph_roman_hundreds = NEMO_DIGIT**3 @ ( graph_roman_hundreds + pynini.union(graph_roman_ties, graph_roman_zero_digit) + pynini.union(graph_roman_digits, graph_roman_zero_digit) ) - graph_roman_ties = NEMO_DIGIT ** 2 @ ( + graph_roman_ties = NEMO_DIGIT**2 @ ( graph_roman_ties + pynini.union(graph_roman_digits, graph_roman_zero_digit) ) graph_roman_digits = NEMO_DIGIT @ graph_roman_digits diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py index c204bcc38..5dd5e175c 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/telephone.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.fr.graph_utils import NEMO_NOT_QUOTE, GraphFst + class TelephoneFst(GraphFst): """ Finite state transducer for verbalizing telephone, e.g. telephone { number_part: "02 33 43 53 22" } - -> 02 33 43 53 22 + -> 02 33 43 53 22 """ def __init__(self): diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/time.py index 93e4886e5..99f5b99e8 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_DIGIT, GraphFst, @@ -20,7 +22,6 @@ delete_space, ) from nemo_text_processing.inverse_text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -28,7 +29,7 @@ class TimeFst(GraphFst): Finite state transducer for verbalizing time, e.g. time { hours: "8" minutes: "30" suffix: "du matin"} -> 8 h 30 time { hours: "8" minutes: "30" } -> 8 h 30 - time { hours: "8" minutes: "30" suffix: "du soir"} -> 20 h 30 + time { hours: "8" minutes: "30" suffix: "du soir"} -> 20 h 30 """ def __init__(self): diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py index 9753b7855..677386d28 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/verbalize_final.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, delete_extra_space, delete_space from nemo_text_processing.inverse_text_normalization.fr.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.fr.verbalizers.word import WordFst -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now """ diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py index 6b350062d..38b1a962c 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/whitelist.py @@ -14,13 +14,14 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space, ) -from pynini.lib import pynutil class WhiteListFst(GraphFst): @@ -38,5 +39,5 @@ def __init__(self): + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete("\"") ) - graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/word.py index fa8f0f39d..6b43ec999 100644 --- a/nemo_text_processing/inverse_text_normalization/fr/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/fr/verbalizers/word.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.fr.graph_utils import ( NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space, ) -from pynini.lib import pynutil class WordFst(GraphFst): @@ -32,6 +33,6 @@ def __init__(self): super().__init__(name="word", kind="verbalize") chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/__init__.py b/nemo_text_processing/inverse_text_normalization/he/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/data/__init__.py b/nemo_text_processing/inverse_text_normalization/he/data/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/data/decimals/__init__.py b/nemo_text_processing/inverse_text_normalization/he/data/decimals/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/decimals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/data/decimals/minutes_exception.tsv b/nemo_text_processing/inverse_text_normalization/he/data/decimals/minutes_exception.tsv new file mode 100644 index 000000000..5626b7100 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/decimals/minutes_exception.tsv @@ -0,0 +1,2 @@ +חצי +רבע diff --git a/nemo_text_processing/inverse_text_normalization/he/data/measurements.tsv b/nemo_text_processing/inverse_text_normalization/he/data/measurements.tsv new file mode 100644 index 000000000..fbd061bc5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/measurements.tsv @@ -0,0 +1,45 @@ +°F פרנהייט +°C צלסיוס +° מעלות +°F מעלות פרנהייט +°C מעלות צלסיוס +K קלווין +% אחוז +% אחוזים +Hz הרץ +kW קילוואט +kW קילו ואט +kW קילו וואט +kWh קילו ואט לשעה +kWh קילוואט לשעה +Wh ואט לשעה +W ואט +ghz ג׳יגה הרץ +ghz גיגה הרץ +khz קילו הרץ +mhz מגה הרץ +v וולט +nm ננומטר +mA מילי אמפר +tW טרה ואט +mv מילי וולט +mW מגה ואט +μm מיקרומטר +" אינץ׳ +cc סי סי +ω אוהם +db דציבל +db דציבלים +kb קילו ביט +mb מגה ביט +gb ג׳יגה ביט +gb גיגה ביט +tb טרה ביט +pb פטה ביט +mb מגה בייט +kb קילו בייט +gb ג׳יגה בייט +gb גיגה בייט +tb טרה בייט +pb פטה בייט +A אמפר \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/months.tsv b/nemo_text_processing/inverse_text_normalization/he/data/months.tsv new file mode 100644 index 000000000..05415cc3d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/months.tsv @@ -0,0 +1,13 @@ +ינואר +פברואר +מרץ +מרס +אפריל +מאי +יוני +יולי +אוגוסט +ספטמבר +אוקטובר +נובמבר +דצמבר \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/months_name2number.tsv b/nemo_text_processing/inverse_text_normalization/he/data/months_name2number.tsv new file mode 100644 index 000000000..651118ca1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/months_name2number.tsv @@ -0,0 +1,12 @@ +ינואר 1 +פברואר 2 +מרץ 3 +אפריל 4 +מאי 5 +יוני 6 +יולי 7 +אוגוסט 8 +ספטמבר 9 +אוקטובר 10 +נובמבר 11 +דצמבר 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/months_ordinal2number.tsv b/nemo_text_processing/inverse_text_normalization/he/data/months_ordinal2number.tsv new file mode 100644 index 000000000..e75a452d8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/months_ordinal2number.tsv @@ -0,0 +1,12 @@ +ראשון 1 +שני 2 +שלישי 3 +רביעי 4 +חמישי 5 +שישי 6 +שביעי 7 +שמיני 8 +תשיעי 9 +עשירי 10 +אחת עשרה 11 +שתיים עשרה 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/he/data/numbers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/decimal_fractions.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/decimal_fractions.tsv new file mode 100644 index 000000000..d88316454 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/decimal_fractions.tsv @@ -0,0 +1,6 @@ +חצי 5 +רבע 25 +שלושת רבעי 75 +עשירית 1 +שתי עשיריות 2 +חמישית 2 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/digit.tsv new file mode 100644 index 000000000..68c02dd42 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/digit.tsv @@ -0,0 +1,20 @@ +אחד 1 +שניים 2 +שני 2 +שלושה 3 +ארבעה 4 +חמישה 5 +שישה 6 +שבעה 7 +שמונה 8 +תשעה 9 +אחת 1 +שתיים 2 +שתים 2 +שתי 2 +שלוש 3 +ארבע 4 +חמש 5 +שש 6 +שבע 7 +תשע 9 diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/hundreds_exception.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/hundreds_exception.tsv new file mode 100644 index 000000000..88e54ab57 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/hundreds_exception.tsv @@ -0,0 +1,2 @@ +מאה 1 +מאתיים 2 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/millions_exception.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/millions_exception.tsv new file mode 100644 index 000000000..1443e5def --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/millions_exception.tsv @@ -0,0 +1 @@ +מיליון 1 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/teen.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/teen.tsv new file mode 100644 index 000000000..26f1a5a4c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/teen.tsv @@ -0,0 +1,21 @@ +עשר 10 +אחד עשר 11 +שניים עשר 12 +שלושה עשר 13 +ארבעה עשר 14 +חמישה עשר 15 +שישה עשר 16 +שבעה עשר 17 +שמונה עשר 18 +תשעה עשר 19 +עשרה 10 +אחת עשרה 11 +שתיים עשרה 12 +שתים עשרה 12 +שלוש עשרה 13 +ארבע עשרה 14 +חמש עשרה 15 +שש עשרה 16 +שבע עשרה 17 +שמונה עשרה 18 +תשע עשרה 19 diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands.tsv new file mode 100644 index 000000000..534789509 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands.tsv @@ -0,0 +1,8 @@ +שלושת 3 +ארבעת 4 +חמשת 5 +ששת 6 +שבעת 7 +שמונת 8 +תשעת 9 +עשרת 10 diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands_exception.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands_exception.tsv new file mode 100644 index 000000000..dd0c71c0d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/thousands_exception.tsv @@ -0,0 +1,2 @@ +אלף 1 +אלפיים 2 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/ties.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/ties.tsv new file mode 100644 index 000000000..b6dd59ca3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +עשרים 2 +שלושים 3 +ארבעים 4 +חמישים 5 +שישים 6 +שבעים 7 +שמונים 8 +תשעים 9 diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/viable_hours.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/viable_hours.tsv new file mode 100644 index 000000000..6a2cb1307 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/viable_hours.tsv @@ -0,0 +1,15 @@ +אחד 1 +אחת 1 +שתיים 2 +שתים 2 +שלוש 3 +ארבע 4 +חמש 5 +שש 6 +שבע 7 +שמונה 8 +תשע 9 +עשר 10 +אחת עשרה 11 +שתיים עשרה 12 +שתים עשרה 12 diff --git a/nemo_text_processing/inverse_text_normalization/he/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/he/data/numbers/zero.tsv new file mode 100644 index 000000000..a0b033c5d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/numbers/zero.tsv @@ -0,0 +1 @@ +אפס 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/ordinals/__init__.py b/nemo_text_processing/inverse_text_normalization/he/data/ordinals/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/data/ordinals/digit.tsv b/nemo_text_processing/inverse_text_normalization/he/data/ordinals/digit.tsv new file mode 100644 index 000000000..036e1433a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/ordinals/digit.tsv @@ -0,0 +1,10 @@ +ראשון אחד +שני שניים +שלישי שלושה +רביעי ארבעה +חמישי חמישה +שישי שישה +שביעי שבעה +שמיני שמונה +תשיעי תשעה +עשירי עשרה \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/prefix.tsv b/nemo_text_processing/inverse_text_normalization/he/data/prefix.tsv new file mode 100644 index 000000000..988d6aedf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/prefix.tsv @@ -0,0 +1,17 @@ +וה +שה +ב +כ +ל +מ +ה +ו +וב +ול +ש +מה +ומ +שכ +שב +בכ +לכ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/spaced_measurements.tsv b/nemo_text_processing/inverse_text_normalization/he/data/spaced_measurements.tsv new file mode 100644 index 000000000..a97b03412 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/spaced_measurements.tsv @@ -0,0 +1,17 @@ +ק״מ קילומטר +ק״מ קילומטרים +מ׳ מטר +מ׳ מטרים +ס״מ סנטימטר +ס״מ סנטימטרים +מ״מ מילימטר +מ״מ מילימטרים +מ״ג מיליגרם +מ״ג מיליגרמים +מ״ל מיליליטר +ק״ג קילוגרם +ק״ג קילוגרמים +קמ״ש קילומטר לשעה +קמ״ש קילומטרים לשעה +ג׳ גרם +ג׳ גרמים \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/__init__.py b/nemo_text_processing/inverse_text_normalization/he/data/time/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/day_suffix.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/day_suffix.tsv new file mode 100644 index 000000000..a4f9d2d46 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/day_suffix.tsv @@ -0,0 +1,2 @@ +בבוקר +לפנות בוקר \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/evening_suffix.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/evening_suffix.tsv new file mode 100644 index 000000000..583470a05 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/evening_suffix.tsv @@ -0,0 +1,2 @@ +בערב +לפנות ערב \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_evening.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_evening.tsv new file mode 100644 index 000000000..4fd47d1e2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_evening.tsv @@ -0,0 +1,7 @@ +5 17 +6 18 +7 19 +8 20 +9 21 +10 22 +11 23 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_night.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_night.tsv new file mode 100644 index 000000000..656d161b2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_night.tsv @@ -0,0 +1,9 @@ +8 20 +9 21 +10 22 +11 23 +12 0 +1 1 +2 2 +3 3 +4 4 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_noon.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_noon.tsv new file mode 100644 index 000000000..8d0de9024 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/hour_to_noon.tsv @@ -0,0 +1,7 @@ +12 12 +1 13 +2 14 +3 15 +4 16 +5 17 +6 18 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/midnight_to_hour.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/midnight_to_hour.tsv new file mode 100644 index 000000000..5b86a39eb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/midnight_to_hour.tsv @@ -0,0 +1 @@ +חצות 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/minute_to.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/minute_to.tsv new file mode 100644 index 000000000..38858859c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/minute_to.tsv @@ -0,0 +1,58 @@ +02 58 +03 57 +04 56 +05 55 +06 54 +07 53 +08 52 +09 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 09 +52 08 +53 07 +54 06 +55 05 +56 04 +57 03 +58 01 +59 01 diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/minute_to_verbose.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/minute_to_verbose.tsv new file mode 100644 index 000000000..8f62ae4de --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/minute_to_verbose.tsv @@ -0,0 +1,6 @@ +רבע 45 +עשרה 50 +חמישה 55 +עשרים 40 +עשרים וחמישה 35 +דקה 59 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/minute_verbose.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/minute_verbose.tsv new file mode 100644 index 000000000..efa2207c3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/minute_verbose.tsv @@ -0,0 +1,8 @@ +שלושת רבעי 45 +חצי 30 +רבע 15 +עשרים 20 +עשרה 10 +חמישה 05 +דקה 01 +שתי 02 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/night_suffix.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/night_suffix.tsv new file mode 100644 index 000000000..464aa81c0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/night_suffix.tsv @@ -0,0 +1 @@ +בלילה \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/noon_suffix.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/noon_suffix.tsv new file mode 100644 index 000000000..963d81053 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/noon_suffix.tsv @@ -0,0 +1,3 @@ +בצהריים +אחרי הצהריים +אחר הצהריים \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/time_suffix.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/time_suffix.tsv new file mode 100644 index 000000000..b5799a0b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/time_suffix.tsv @@ -0,0 +1,8 @@ +בבוקר +לפנות בוקר +לפנות ערב +בערב +בצהריים +בלילה +אחרי הצהריים +אחר הצהריים \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/time/to_hour.tsv b/nemo_text_processing/inverse_text_normalization/he/data/time/to_hour.tsv new file mode 100644 index 000000000..5689943fd --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/time/to_hour.tsv @@ -0,0 +1,13 @@ +אחת 12 +שתיים 1 +שלוש 2 +ארבע 3 +חמש 4 +שש 5 +שבע 6 +שמונה 7 +תשע 8 +עשר 9 +אחת עשרה 10 +שתיים עשרה 11 +חצות 23 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/data/whitelist.tsv b/nemo_text_processing/inverse_text_normalization/he/data/whitelist.tsv new file mode 100644 index 000000000..9844685c5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/data/whitelist.tsv @@ -0,0 +1,20 @@ +אח״כ אחר כך +וכו׳ וכולי +בריה״מ ברית המועצות +ארה״ב ארצות הברית +עו״ד עורך דין +עו״ד עורכת דין +עו״ד עורכי דין +עו״ד עורכות דין +רו״ח רואה חשבון +רו״ח רואת חשבון +רו״ח רואי חשבון +רו״ח רואות חשבון +לפנה״ס לפני הספירה +ד״ר דוקטור +פרופ׳ פרופסור +אמא אימא +כל כול +מאיתנו מאתנו +ישארו יישארו +יתכן ייתכן \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/he/graph_utils.py b/nemo_text_processing/inverse_text_normalization/he/graph_utils.py new file mode 100644 index 000000000..072da0381 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/graph_utils.py @@ -0,0 +1,119 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +import pynini +from pynini import Far +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, delete_space +from nemo_text_processing.text_normalization.en.utils import load_labels + +NEMO_ALPHA_HE = pynini.union(*"אבגדהוזחטיכלמםנןסעפףצץקרשת").optimize() +delete_and = pynutil.delete("ו") +delete_optional_and = delete_and.ques + +#################### +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +MINUS = pynini.union("מינוס").optimize() + + +def string_map_cased(input_file: str): + labels = load_labels(input_file) + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/he/taggers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/he/taggers/cardinal.py new file mode 100644 index 000000000..aaf30b32c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/cardinal.py @@ -0,0 +1,158 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import ( + NEMO_ALPHA_HE, + GraphFst, + delete_and, + delete_optional_and, +) +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals in Hebrew + e.g. מינוס עשרים ושלוש ("minus twenty three" in Hebrew)-> cardinal { negative: "-" integer: "23" } } + """ + + def __init__(self): + super().__init__(name="cardinal", kind="classify") + + # digits + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + prefix_graph = pynini.string_file(get_abs_path("data/prefix.tsv")) + + # teens + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_ties += pynini.union( + delete_space + delete_optional_and + graph_digit, + pynutil.insert("0", weight=0.001), + ) + graph_two_digit = pynini.union(graph_teen, graph_ties) + + self.graph_two_digit = pynini.union(graph_digit, graph_ties, pynutil.add_weight(graph_teen, -0.001)) + + # hundreds + hundred_exception = pynini.string_file(get_abs_path("data/numbers/hundreds_exception.tsv")) + delete_hundred = pynutil.delete("מאות") + graph_hundred = delete_optional_and + pynini.union( + hundred_exception, + graph_digit + delete_space + delete_hundred, + pynutil.insert("0", weight=0.001), + ) + graph_hundred += delete_space + graph_hundred += pynini.union( + delete_optional_and + graph_two_digit, + pynutil.insert("0") + delete_space + delete_and + graph_digit, + pynutil.insert("00", weight=0.001), + ) + graph_hundred = pynini.union( + graph_hundred, + pynutil.insert("0") + graph_two_digit, + pynutil.insert("00") + graph_digit, + ) + self.graph_hundred = graph_hundred @ ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) + ) + + # thousands + thousand_exception = pynini.string_file(get_abs_path("data/numbers/thousands_exception.tsv")) + thousand_digit = pynini.string_file(get_abs_path("data/numbers/thousands.tsv")) + delete_thousand = pynutil.delete("אלפים") | pynutil.delete("אלף", weight=0.001) + + large_number_prefix = pynini.union( + graph_hundred, + pynutil.insert("0") + graph_two_digit, + pynutil.insert("00") + thousand_digit, + ) + many_thousands = large_number_prefix + delete_space + delete_thousand + graph_thousands = delete_optional_and + pynini.union( + (pynutil.insert("00") + thousand_exception), + many_thousands, + pynutil.insert("000", weight=0.001), + ) + + self.graph_thousands = pynini.union(graph_thousands + delete_space + graph_hundred, graph_zero) + self.graph_thousands @= pynini.union( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), + "0", + ) + + # millions + million_exceptions = pynini.string_file(get_abs_path("data/numbers/millions_exception.tsv")) + million_exceptions = pynutil.insert("00") + million_exceptions + delete_millions = pynutil.delete("מיליונים") | pynutil.delete("מיליון", weight=0.001) + many_millions = large_number_prefix + delete_space + delete_millions + graph_millions = pynini.union(many_millions, million_exceptions, pynutil.insert("000", weight=0.001)) + + graph = pynini.union( + graph_millions + delete_space + graph_thousands + delete_space + graph_hundred, + graph_zero, + ) + graph = graph @ pynini.union( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), + "0", + ) + + labels_exception = load_labels(get_abs_path("data/numbers/digit.tsv")) + labels_exception = list(set([x[0] for x in labels_exception] + ["אפס", "עשר", "עשרה"])) + labels_exception += ["ו" + label for label in labels_exception] + graph_exception = pynini.union(*labels_exception).optimize() + graph = ((NEMO_ALPHA_HE + NEMO_SIGMA) @ graph).optimize() + + self.graph_no_exception = graph + + ### Token insertion + minus_graph = pynutil.insert("negative: ") + pynini.cross("מינוס", '"-"') + NEMO_SPACE + optional_minus_graph = pynini.closure(minus_graph, 0, 1) + + optional_prefix_graph = pynini.closure( + pynutil.insert('morphosyntactic_features: "') + prefix_graph + pynutil.insert('"') + insert_space, + 0, + 1, + ) + + graph_wo_small_digits = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + + cardinal_wo_viable_hours = load_labels(get_abs_path("data/numbers/viable_hours.tsv")) + cardinal_wo_viable_hours = list(set([x[0] for x in cardinal_wo_viable_hours])) + viable_hours_exception = pynini.union(*cardinal_wo_viable_hours).optimize() + self.graph_wo_viable_hours = (pynini.project(graph, "input") - viable_hours_exception.arcsort()) @ graph + + small_number_with_minus = ( + insert_space + minus_graph + pynutil.insert('integer: "') + self.graph_no_exception + pynutil.insert('"') + ) + + big_number_with_optional_minus = ( + optional_minus_graph + pynutil.insert('integer: "') + graph_wo_small_digits + pynutil.insert('"') + ) + + graph = optional_prefix_graph + (small_number_with_minus | big_number_with_optional_minus) + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/date.py b/nemo_text_processing/inverse_text_normalization/he/taggers/date.py new file mode 100644 index 000000000..cf9cacbd5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/date.py @@ -0,0 +1,106 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import delete_extra_space, delete_space, insert_space + + +def _get_year_graph(graph_two_digits, graph_thousands): + """ + Transducer for year, e.g. twenty twenty -> 2020 + """ + year_graph = pynini.union( + (graph_two_digits + delete_space + graph_two_digits), + graph_thousands, # 20 19, 40 12, 20 20 + ) # 2012 - assuming no limit on the year + + year_graph.optimize() + return year_graph + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date in Hebrew, + e.g. אחד במאי אלף תשע מאות שמונים ושלוש -> date { day: "1" morphosyntactic_features: "ב" month: "5" year: "1983" } + e.g. מרץ אלף תשע מאות שמונים ותשע -> date { month: "מרץ" year: "1989" } + e.g. בינואר עשרים עשרים -> date { morphosyntactic_features: "ב" month: "ינואר" year: "2020" } + + Args: + cardinal: CardinalFst + ordinal: OrdinalFst + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst): + super().__init__(name="date", kind="classify") + + ordinal_graph = ordinal.graph + two_digits_graph = cardinal.graph_two_digit + + day_graph = pynutil.add_weight(two_digits_graph | ordinal_graph, -0.7) + day_graph = pynutil.insert('day: "') + day_graph + pynutil.insert('"') + + month_names = pynini.string_file(get_abs_path("data/months.tsv")) + month_names_graph = pynutil.insert('month: "') + month_names + pynutil.insert('"') + + month_name2number = pynini.string_file(get_abs_path("data/months_name2number.tsv")) + month_name2number_graph = pynutil.insert('month: "') + month_name2number + pynutil.insert('"') + + month_number2number = pynini.string_file(get_abs_path("data/months_ordinal2number.tsv")) + month_number2number_graph = pynutil.insert('month: "') + month_number2number + pynutil.insert('"') + + all_month_graph = month_name2number_graph | month_number2number_graph + + year_graph = _get_year_graph(two_digits_graph, cardinal.graph_thousands) + graph_year = delete_extra_space + pynutil.insert('year: "') + year_graph + pynutil.insert('"') + + prefix_graph = pynini.string_file(get_abs_path("data/prefix.tsv")) + delete_prefix = pynutil.delete(prefix_graph) + + graph_prefix = pynutil.insert('morphosyntactic_features: "') + prefix_graph + pynutil.insert('"') + year_prefix_graph = ( + pynutil.insert('morphosyntactic_features: "') + + pynini.closure(prefix_graph, 0, 1) + + pynini.union("שנה", "שנת") + + pynutil.insert('"') + ) + + graph_dm = ( + pynini.closure(graph_prefix + insert_space, 0, 1) + + day_graph + + insert_space + + delete_space + + pynini.closure(delete_prefix + insert_space, 0, 1) + + month_name2number_graph + ) + + graph_dmy = ( + pynini.closure(graph_prefix + insert_space, 0, 1) + + day_graph + + insert_space + + delete_space + + pynini.closure(delete_prefix + insert_space, 0, 1) + + all_month_graph + + graph_year + ) + + graph_my = pynini.closure(graph_prefix + insert_space, 0, 1) + month_names_graph + graph_year + graph_y_only = year_prefix_graph + graph_year + + final_graph = graph_dm | graph_dmy | graph_my | graph_y_only + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/he/taggers/decimal.py new file mode 100644 index 000000000..ecefb306a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/decimal.py @@ -0,0 +1,144 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import MINUS, GraphFst, delete_and +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + delete_extra_space, + delete_space, + delete_zero_or_one_space, + insert_space, +) + + +def get_quantity(decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike") -> "pynini.FstLike": + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral in Hebrew, + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + numbers = cardinal_up_to_hundred @ ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + + suffix_labels = ["מיליון", "מיליארד"] + suffix = pynini.union(*suffix_labels).optimize() + + res = ( + pynutil.insert('integer_part: "') + + numbers + + pynutil.insert('"') + + delete_extra_space + + pynutil.insert('quantity: "') + + suffix + + pynutil.insert('"') + ) + res |= decimal + delete_extra_space + pynutil.insert('quantity: "') + (suffix | "אלף") + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal in Hebrew + e.g. עשרים ושלוש וחצי -> decimal { integer_part: "23" fractional_part: "5" } + e.g. אחד נקודה שלוש -> decimal { integer_part: "1" fractional_part: "3" } + e.g. ארבע נקודה חמש מיליון -> decimal { integer_part: "4" fractional_part: "5" quantity: "מיליון" } + e.g. מינוס ארבע מאות נקודה שלוש שתיים שלוש -> decimal { negative: "true" integer_part: "400" fractional_part: "323" } + e.g. אפס נקודה שלושים ושלוש -> decimal { integer_part: "0" fractional_part: "33" } + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="decimal", kind="classify") + + prefix_graph = pynini.string_file(get_abs_path("data/prefix.tsv")) + optional_prefix_graph = pynini.closure( + pynutil.insert('morphosyntactic_features: "') + prefix_graph + pynutil.insert('"') + insert_space, + 0, + 1, + ) + + # all cardinals + cardinal_graph = cardinal.graph_no_exception + + # all fractions + fractions = pynini.string_file(get_abs_path("data/numbers/decimal_fractions.tsv")) + fractions_graph = delete_zero_or_one_space + delete_and + fractions + fractions_graph = pynutil.insert('fractional_part: "') + fractions_graph + pynutil.insert('"') + + # identify decimals that can be understood as time, don't convert them to avoid ambiguity + viable_minutes_exception = pynini.string_file(get_abs_path("data/decimals/minutes_exception.tsv")) + fractions_wo_minutes = (pynini.project(fractions, "input") - viable_minutes_exception.arcsort()) @ fractions + fractions_wo_minutes = delete_zero_or_one_space + delete_and + fractions_wo_minutes + fractions_wo_minutes = pynutil.insert('fractional_part: "') + fractions_wo_minutes + pynutil.insert('"') + + graph_decimal = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_decimal |= cardinal.graph_two_digit + graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal + self.graph = graph_decimal + + point = pynutil.delete("נקודה") + + graph_negative = pynutil.insert("negative: ") + pynini.cross(MINUS, '"true"') + delete_extra_space + optional_graph_negative = pynini.closure( + graph_negative, + 0, + 1, + ) + + graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + graph_fractional = pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"') + + # integer could be an hour, but minutes cannot: convert to decimal + viable_hour_unviable_minutes = graph_integer + delete_extra_space + fractions_wo_minutes + + # integer cannot be an hour, but minutes can: convert to decimal + unviable_hour_viable_minutes = ( + pynutil.insert('integer_part: "') + + cardinal.graph_wo_viable_hours + + pynutil.insert('"') + + delete_extra_space + + fractions_graph + ) + + # minus sign followed by ambiguous decimal: convert to decimal, there is no negative time + negative_viable_time = graph_negative + graph_integer + delete_extra_space + fractions_graph + + # all decimals with fractions, not excluding anything (used in other FSTs) + all_decimals_wo_point = graph_integer + delete_extra_space + fractions_graph + + # only cases with fractional part that cannot be interpreted as time + graph_wo_point = viable_hour_unviable_minutes | unviable_hour_viable_minutes | negative_viable_time + + # all decimals with the word "point" + graph_w_point = ( + pynini.closure(graph_integer + delete_extra_space, 0, 1) + point + delete_extra_space + graph_fractional + ) + + final_graph_wo_sign = graph_w_point | graph_wo_point + self.final_graph_wo_sign = graph_w_point | all_decimals_wo_point + final_graph = optional_prefix_graph + optional_graph_negative + final_graph_wo_sign + + quantity_graph = get_quantity(self.final_graph_wo_sign, cardinal.graph_hundred) + final_graph |= optional_prefix_graph + optional_graph_negative + quantity_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/he/taggers/measure.py new file mode 100644 index 000000000..0232c4ff6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/measure.py @@ -0,0 +1,114 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.he.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.he.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_SPACE, + delete_extra_space, + delete_space, + delete_zero_or_one_space, + insert_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure in Hebrew + e.g. מש עשרה אחוז -> measure { cardinal { integer: "15" } units: "%" } + e.g. מינוס חמש עשרה אחוז -> measure { cardinal { negative: "-" integer: "15" } units: "%" } + e.g. שלוש מיליגרם -> measure { cardinal { integer: "3" } units: "מ״ג" } + e.g. אלף אחוז -> measure { cardinal { integer: "1000" } units: "%" } + e.g. אחוז אחד -> measure { units: "%" cardinal { integer: "1" } } + e.g. סנטימטר אחד -> measure { units: "ס״מ" cardinal { integer: "1" } } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: CardinalFst, decimal: DecimalFst): + super().__init__(name="measure", kind="classify") + + # optional negative sign + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("מינוס", '"-"') + NEMO_SPACE, + 0, + 1, + ) + + prefix_graph = pynini.string_file(get_abs_path("data/prefix.tsv")) + optional_prefix_graph = pynini.closure( + pynutil.insert('morphosyntactic_features: "') + prefix_graph + pynutil.insert('"') + insert_space, + 0, + 1, + ) + + # cardinal numbers + cardinal_graph = cardinal.graph_no_exception + + # Let singular apply to values > 1 as they could be part of an adjective phrase (e.g. 14 foot tall building) + subgraph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal.final_graph_wo_sign + + pynutil.insert(" }") + + delete_extra_space + ) + + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert('integer: "') + + cardinal_graph + + pynutil.insert('"') + + pynutil.insert(" }") + + delete_extra_space + ) + + # convert units + joined_units = pynini.string_file(get_abs_path("data/measurements.tsv")) + joined_units = pynini.invert(joined_units) + joined_units = pynutil.insert('units: "') + joined_units + pynutil.insert('"') + + spaced_units = pynini.string_file(get_abs_path("data/spaced_measurements.tsv")) + spaced_units = pynini.invert(spaced_units) + spaced_units = pynutil.insert('units: "\[SPACE\]') + spaced_units + pynutil.insert('"') # noqa: W605 + + # in joint units the unit is concatenated to the number, in spaced unit separate the unit with a space + units_graph = joined_units | spaced_units + + # one graph is needed since it changed the order of the words. + # We say "ten percent" for 10% but "percent one" for 1% + one = pynini.string_map([("אחד", "1")]) + one_graph = ( + insert_space + + pynutil.insert("cardinal { ") + + pynutil.insert('integer: "') + + one + + pynutil.insert('"') + + pynutil.insert(" }") + ) + + number_graph = subgraph_decimal | subgraph_cardinal + number_unit_graph = (number_graph + units_graph) | (units_graph + delete_space + one_graph) + + final_graph = optional_prefix_graph + number_unit_graph + delete_zero_or_one_space + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/he/taggers/ordinal.py new file mode 100644 index 000000000..c7306ea43 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/ordinal.py @@ -0,0 +1,43 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal in Hebrew + e.g. ראשון -> ordinal { integer: "1" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="ordinal", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")) + graph = NEMO_SIGMA + graph_digit + + self.graph = graph @ cardinal_graph + + final_graph = pynutil.insert('integer: "') + self.graph + pynutil.insert('"') + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/he/taggers/punctuation.py new file mode 100644 index 000000000..b963e7b74 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/punctuation.py @@ -0,0 +1,35 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&'()*+,-./:;<=>?@^_`{|}~" + punct = pynini.union(*s) + + graph = pynutil.insert('name: "') + punct + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/time.py b/nemo_text_processing/inverse_text_normalization/he/taggers/time.py new file mode 100644 index 000000000..ac4965cfc --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/time.py @@ -0,0 +1,202 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst, delete_and +from nemo_text_processing.inverse_text_normalization.he.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path, integer_to_text +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + delete_extra_space, + delete_space, + delete_zero_or_one_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time in Hebrew. + Conversion is made only when am / pm time is not ambiguous! + e.g. שלוש דקות לחצות -> time { minutes: "57" hours: "23" } + e.g. באחת ושתי דקות בצהריים -> time { morphosyntactic_features: "ב" hours: "1" minutes: "02" suffix: "צהריים" } + e.g. שתיים ועשרה בבוקר -> time { hours: "2" minutes: "10" suffix: "בוקר" } + e.g. שתיים ועשרה בצהריים -> time { hours: "2" minutes: "10" suffix: "צהריים" } + e.g. שתיים עשרה ושלוש דקות אחרי הצהריים -> time { hours: "12" minutes: "03" suffix: "צהריים" } + e.g. רבע לשש בערב -> time { minutes: "45" hours: "5" suffix: "ערב" } + + """ + + def __init__(self): + super().__init__(name="time", kind="classify") + + # hours, minutes, seconds, suffix, zone, style, speak_period + midnight_to_hour_graph = pynini.string_file(get_abs_path("data/time/midnight_to_hour.tsv")) + to_hour_graph = pynini.string_file(get_abs_path("data/time/to_hour.tsv")) + + minute_verbose_graph = pynini.string_file(get_abs_path("data/time/minute_verbose.tsv")) + minute_to_graph = pynini.string_file(get_abs_path("data/time/minute_to.tsv")) + minute_to_verbose_graph = pynini.string_file(get_abs_path("data/time/minute_to_verbose.tsv")) + + suffix_graph = pynini.union( + pynini.string_file(get_abs_path("data/time/day_suffix.tsv")), + pynini.string_file(get_abs_path("data/time/noon_suffix.tsv")), + pynini.string_file(get_abs_path("data/time/evening_suffix.tsv")), + pynini.string_file(get_abs_path("data/time/night_suffix.tsv")), + ) + + time_prefix = pynini.string_file(get_abs_path("data/prefix.tsv")) + time_prefix_graph = ( + pynutil.insert('morphosyntactic_features: "') + time_prefix + pynutil.insert('"') + insert_space + ) + optional_time_prefix_graph = pynini.closure(time_prefix_graph, 0, 1) + + # only used for < 1000 thousand -> 0 weight + cardinal = pynutil.add_weight(CardinalFst().graph_no_exception, weight=-0.7) + + labels_hour = [integer_to_text(x, only_fem=True)[0] for x in range(1, 13)] + labels_minute_single = [integer_to_text(x, only_fem=True)[0] for x in range(2, 10)] + labels_minute_double = [integer_to_text(x, only_fem=True)[0] for x in range(10, 60)] + + graph_hour = pynini.union(*labels_hour) @ cardinal + graph_hour |= midnight_to_hour_graph + add_leading_zero_to_double_digit = pynutil.insert("0") + NEMO_DIGIT + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal @ add_leading_zero_to_double_digit + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal + + final_graph_hour = pynutil.insert('hours: "') + graph_hour + pynutil.insert('"') + + graph_minute = pynini.union(pynutil.insert("00"), graph_minute_single, graph_minute_double) + + final_suffix = pynutil.insert('suffix: "') + suffix_graph + pynutil.insert('"') + final_suffix = delete_space + insert_space + final_suffix + + time_word = "דקות" + optional_delete_time = pynini.closure(delete_space + pynutil.delete(time_word), 0, 1) + graph_h_and_m = ( + final_graph_hour + + delete_space + + delete_and + + insert_space + + pynutil.insert('minutes: "') + + pynini.union(graph_minute_single, graph_minute_double, minute_verbose_graph) + + pynutil.insert('"') + + optional_delete_time + ) + + graph_special_m_to_h_suffix_time = ( + pynutil.insert('minutes: "') + + minute_to_verbose_graph + + pynutil.insert('"') + + delete_space + + pynutil.delete("ל") + + insert_space + + pynutil.insert('hours: "') + + to_hour_graph + + pynutil.insert('"') + ) + + graph_m_to_h_suffix_time = ( + pynutil.insert('minutes: "') + + pynini.union(graph_minute_single, graph_minute_double) @ minute_to_graph + + pynutil.insert('"') + + optional_delete_time + + delete_space + + pynutil.delete("ל") + + insert_space + + pynutil.insert('hours: "') + + to_hour_graph + + pynutil.insert('"') + ) + + graph_h = ( + optional_time_prefix_graph + + delete_zero_or_one_space + + final_graph_hour + + delete_extra_space + + pynutil.insert('minutes: "') + + (pynutil.insert("00") | graph_minute) + + pynutil.insert('"') + + final_suffix + ) + + midnight_graph = ( + optional_time_prefix_graph + + delete_zero_or_one_space + + pynutil.insert('hours: "') + + midnight_to_hour_graph + + pynutil.insert('"') + + insert_space + + pynutil.insert('minutes: "') + + (pynutil.insert("00") | graph_minute) + + pynutil.insert('"') + ) + + graph_midnight_and_m = ( + pynutil.insert('hours: "') + + midnight_to_hour_graph + + pynutil.insert('"') + + delete_space + + delete_and + + insert_space + + pynutil.insert('minutes: "') + + pynini.union(graph_minute_single, graph_minute_double, minute_verbose_graph) + + pynutil.insert('"') + + optional_delete_time + ) + + to_midnight_verbose_graph = ( + pynutil.insert('minutes: "') + + minute_to_verbose_graph + + pynutil.insert('"') + + delete_space + + pynutil.delete("ל") + + insert_space + + pynutil.insert('hours: "') + + to_hour_graph + + pynutil.insert('"') + ) + + graph_m_to_midnight = ( + pynutil.insert('minutes: "') + + pynini.union(graph_minute_single, graph_minute_double) @ minute_to_graph + + pynutil.insert('"') + + optional_delete_time + + delete_space + + pynutil.delete("ל") + + insert_space + + pynutil.insert('hours: "') + + to_hour_graph + + pynutil.insert('"') + ) + + final_graph_midnight = ( + optional_time_prefix_graph + + delete_zero_or_one_space + + (midnight_graph | to_midnight_verbose_graph | graph_m_to_midnight | graph_midnight_and_m) + ) + + final_graph = ( + optional_time_prefix_graph + + delete_zero_or_one_space + + (graph_h_and_m | graph_special_m_to_h_suffix_time | graph_m_to_h_suffix_time) + + final_suffix + ) + final_graph |= graph_h + final_graph |= final_graph_midnight + + final_graph = self.add_tokens(final_graph.optimize()) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/he/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..807dcf734 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/tokenize_and_classify.py @@ -0,0 +1,104 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.he.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.he.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.he.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.he.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.he.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.he.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.he.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.he.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.he.taggers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import delete_extra_space, delete_space, generator_main + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + input_case: str = None, + ): + + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"he_itn.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst + date_graph = DateFst(ordinal=ordinal, cardinal=cardinal).fst + word_graph = WordFst().fst + time_graph = TimeFst().fst + whitelist_graph = WhiteListFst(input_file=whitelist).fst + punct_graph = PunctuationFst().fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/he/taggers/whitelist.py new file mode 100644 index 000000000..58de7668e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/whitelist.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst, string_map_cased +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import convert_space, insert_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + e.g. misses -> tokens { name: "mrs." } + This class has highest priority among all classifier grammars. + Whitelisted tokens are defined and loaded from "data/whitelist.tsv" (unless input_file specified). + + Args: + input_file: path to a file with whitelist replacements (each line of the file: written_form\tspoken_form\n), + e.g. nemo_text_processing/inverse_text_normalization/he/data/whitelist.tsv + """ + + def __init__(self, input_file: str = None): + super().__init__(name="whitelist", kind="classify") + prefix_graph = pynini.string_file(get_abs_path("data/prefix.tsv")) + + if input_file is None: + input_file = get_abs_path("data/whitelist.tsv") + + if not os.path.exists(input_file): + raise ValueError(f"Whitelist file {input_file} not found") + + optional_prefix_graph = pynini.closure( + pynutil.insert('morphosyntactic_features: "') + prefix_graph + pynutil.insert('"') + insert_space, + 0, + 1, + ) + whitelist = string_map_cased(input_file) + graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"') + final_graph = optional_prefix_graph + graph + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/taggers/word.py b/nemo_text_processing/inverse_text_normalization/he/taggers/word.py new file mode 100644 index 000000000..6b5394ac3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/taggers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. sleep -> tokens { name: "sleep" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"') + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/utils.py b/nemo_text_processing/inverse_text_normalization/he/utils.py new file mode 100644 index 000000000..1aa996b80 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/utils.py @@ -0,0 +1,182 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +#################### +# HEBREW CONSTANTS # +#################### +units_feminine_dict = { + "0": "אפס", + "1": "אחת", + "2": "שתיים", + "3": "שלוש", + "4": "ארבע", + "5": "חמש", + "6": "שש", + "7": "שבע", + "8": "שמונה", + "9": "תשע", +} + +units_masculine_dict = { + "0": "אפס", + "1": "אחד", + "2": "שניים", + "3": "שלושה", + "4": "ארבעה", + "5": "חמישה", + "6": "שישה", + "7": "שבעה", + "8": "שמונה", + "9": "תשעה", +} + +tens_dict = { + "2": "עשרים", + "3": "שלושים", + "4": "ארבעים", + "5": "חמישים", + "6": "שישים", + "7": "שבעים", + "8": "שמונים", + "9": "תשעים", +} + +ten = { + "short": "עשר", + "long": "עשרה", +} # double pronunciation: short is 'eser' and 'asar', long is 'esre' and 'asara' + + +############# +# FUNCTIONS # +############# +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res + + +def digit_by_digit(num): + + dbd = [" ".join([units_feminine_dict[digit] for digit in num])] + + # generate "1" as masculine and as feminine if exists + if units_feminine_dict["1"] in dbd[0]: + dbd.append(dbd[0].replace(units_feminine_dict["1"], units_masculine_dict["1"])) + + return dbd + + +def integer_to_text(num, only_fem=False): + if isinstance(num, int): + num = str(num) + # number is zero + if num == len(num) * "0": + return ["אפס"] + else: + # remove leading zeros from number + num = num.lstrip("0") + + # units + if len(num) == 1: + return _less_than_10(num, only_fem) + + # tenths + elif len(num) == 2: + return _less_than_100(num, only_fem) + + else: + raise Exception + + +def _less_than_10(num, only_fem=False): + """ + Returns a list of all the possible names of a number in range 0-9 + """ + + if only_fem: + return [units_feminine_dict[num]] + else: + return [units_feminine_dict[num], units_masculine_dict[num]] + + +def _less_than_100(num, only_fem=False): + """ + Returns a list of all the possible names of a number in range 0-99 + """ + + # init result + res = list() + + # split number to digits + tens, units = num + + # number is in range 0-9 + if len(num) == 1: + res.extend(_less_than_10(num)) + + # number is in range 10-99 + elif len(num) == 2: + + if num == "10": + if only_fem: + res.extend([ten["short"]]) + else: + res.extend([ten["long"], ten["short"]]) + + # number is in range 11-19 + elif tens == "1": + res.append(f'{units_feminine_dict[num[1]]} {ten["long"]}') + if not only_fem: + res.append(f'{units_masculine_dict[num[1]]} {ten["short"]}') + + else: + + # number is in range 20-99, a multiplication of 10 + if units == "0": + res.append(tens_dict[num[0]]) + + # number is in range 20-99, but not multiplication of 10 + else: + res.append(f'{tens_dict[num[0]]} {"ו"}{units_feminine_dict[num[1]]}') + if not only_fem: + res.append(f'{tens_dict[num[0]]} {"ו"}{units_masculine_dict[num[1]]}') + + return res diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/cardinal.py new file mode 100644 index 000000000..d26e1f703 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/cardinal.py @@ -0,0 +1,84 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import NEMO_ALPHA_HE, GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal in Hebrew + e.g. cardinal { prefix: "וב" integer: "3405"} -> וב-3,405 + e.g. cardinal { negative: "-" integer: "904" } -> -904 + e.g. cardinal { prefix: "כ" integer: "123" } -> כ-123 + + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + # Need parser to group digits by threes + exactly_three_digits = NEMO_DIGIT**3 + at_most_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) + + # Thousands separator + group_by_threes = at_most_three_digits + (pynutil.insert(",") + exactly_three_digits).closure() + + # Keep the prefix if exists and add a dash + optional_prefix = pynini.closure( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_ALPHA_HE, 1) + + pynutil.insert("-") + + pynutil.delete('"') + + delete_space, + 0, + 1, + ) + + # Removes the negative attribute and leaves the sign if occurs + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete('"') + + pynini.accep("-") + + pynutil.delete('"') + + delete_space, + 0, + 1, + ) + + # removes integer aspect + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_DIGIT, 1) # Accepts at least one digit + + pynutil.delete('"') + ) + + # Add thousands separator + graph = graph @ group_by_threes + + self.numbers = graph + + # add prefix and sign + graph = optional_prefix + optional_sign + graph + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/date.py new file mode 100644 index 000000000..4a1b24599 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/date.py @@ -0,0 +1,120 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + delete_space, + delete_zero_or_one_space, + insert_space, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, + e.g. { day_prefix: "ה" day: "1" month_prefix: "ב" month: "6" year: "2012" } -> ה-1.6.2012 + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + + day_prefix = ( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.insert("-") + + pynutil.delete('"') + ) + + day = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1, 2) + + pynutil.insert(".") + + pynutil.delete('"') + + delete_space + ) + + month_prefix = ( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + delete_space + ) + + month = ( + pynutil.delete("month:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + year_prefix = ( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 3) + + pynutil.delete('"') + + delete_space + ) + + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + ####################### + # DATE FORMATS GRAPHS # + ####################### + + # day and month only + graph_dm = ( + pynini.closure(day_prefix + delete_zero_or_one_space, 0, 1) + + day + + pynini.closure(delete_zero_or_one_space, 0, 1) + + month + + delete_zero_or_one_space + ) + + # day month and year + graph_dmy = graph_dm + delete_space + pynutil.insert(".") + delete_zero_or_one_space + year + + # only month and year + graph_my = ( + pynini.closure(month_prefix + delete_zero_or_one_space, 0, 1) + + month + + pynutil.insert(NEMO_SPACE) + + pynini.closure(delete_zero_or_one_space + year, 0, 1) + ) + + # only year + graph_y_only = year_prefix + insert_space + year + + final_graph = (graph_dm | graph_dmy | graph_my | graph_y_only) + delete_space + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/decimal.py new file mode 100644 index 000000000..ea69ab784 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/decimal.py @@ -0,0 +1,90 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import NEMO_ALPHA_HE, GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_NOT_QUOTE, delete_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, + e.g. decimal { integer_part: "0" fractional_part: "33" } -> 0.33 + e.g. decimal { negative: "true" integer_part: "400" fractional_part: "323" } -> -400.323 + e.g. decimal { integer_part: "4" fractional_part: "5" quantity: "מיליון" } -> 4.5 מיליון + + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + optionl_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1) + + # Need parser to group digits by threes + exactly_three_digits = NEMO_DIGIT**3 + at_most_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) + + # Thousands separator + group_by_threes = at_most_three_digits + (pynutil.insert(",") + exactly_three_digits).closure() + + integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + integer = integer @ group_by_threes + + optional_integer = pynini.closure(integer + delete_space, 0, 1) + + fractional = ( + pynutil.insert(".") + + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + optional_fractional = pynini.closure(fractional + delete_space, 0, 1) + + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1) + + # Keep the prefix if exists and add a dash + optional_prefix = pynini.closure( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_ALPHA_HE, 1) + + pynutil.insert("-") + + pynutil.delete('"') + + delete_space, + 0, + 1, + ) + + graph = optional_prefix + optional_integer + optional_fractional + optional_quantity + self.numbers = graph + graph = optionl_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/measure.py new file mode 100644 index 000000000..a4aadd67b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/measure.py @@ -0,0 +1,107 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + delete_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, in Hebrew. + Some measures are concatenated to the numbers and other are don't (two measure lists) + e.g. measure { cardinal { integer: "3" } units: "מ״ג" } -> 3 מ״ג + e.g. measure { cardinal { integer: "1000" } units: "%" } -> 1,000% + e.g. measure { units: "%" cardinal { integer: "1" } } -> 1% + e.g. measure { units: "ס״מ" cardinal { integer: "1" } } -> 1 ס״מ + e.g. measure { prefix: "ל" cardinal { integer: "4" } units: "ס״מ" } -> ל-4 ס״מ + + Args: + decimal: DecimalFst + cardinal: CardinalFst + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst): + super().__init__(name="measure", kind="verbalize") + + optional_prefix = pynini.closure( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.insert("-") + + pynutil.delete('"') + + delete_space, + 0, + 1, + ) + + # Removes the negative attribute and leaves the sign if occurs + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete('"') + + pynini.accep("-") + + pynutil.delete('"') + + delete_space, + 0, + 1, + ) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + + unit = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + + pynutil.delete('"') + + delete_space + ) + unit @= pynini.cdrewrite( + pynini.cross("\[SPACE\]", NEMO_SPACE), "", "", NEMO_SIGMA # noqa: W605 + ) # For space separated measures. + + numbers_units = delete_space + unit + numbers_graph = (graph_cardinal | graph_decimal) + numbers_units + + one_graph = delete_space + pynutil.insert("1") + unit + pynutil.delete('cardinal { integer: "1" }') + + graph = optional_prefix + (numbers_graph | one_graph) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/ordinal.py new file mode 100644 index 000000000..a85f5b019 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/ordinal.py @@ -0,0 +1,38 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal in Hebrew + e.g. ordinal { integer: "10" } -> 10 + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete('"') + ) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/time.py new file mode 100644 index 000000000..3d41b783b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/time.py @@ -0,0 +1,95 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.he.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + delete_space, + delete_zero_or_one_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time in Hebrew + e.g. time { hours: "2" minutes: "55" suffix: "בלילה" } -> 2:55 בלילה + e.g. time { hours: "2" minutes: "57" suffix: "בבוקר" } -> 2:57 בבוקר + e.g. time { morphosyntactic_features: "ב" hours: "6" minutes: "32" suffix: "בערב" } -> ב-18:32 בערב + e.g. time { morphosyntactic_features: "בשעה" hours: "2" minutes: "10" suffix: "בצהריים" } -> בשעה-14:10 בצהריים + + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + + hour_to_noon = pynini.string_file(get_abs_path("data/time/hour_to_noon.tsv")) + hour_to_evening = pynini.string_file(get_abs_path("data/time/hour_to_evening.tsv")) + hour_to_night = pynini.string_file(get_abs_path("data/time/hour_to_night.tsv")) + + day_suffixes = pynini.string_file(get_abs_path("data/time/day_suffix.tsv")) + day_suffixes = insert_space + pynutil.delete('suffix: "') + day_suffixes + pynutil.delete('"') + + noon_suffixes = pynini.string_file(get_abs_path("data/time/noon_suffix.tsv")) + noon_suffixes = insert_space + pynutil.delete('suffix: "') + noon_suffixes + pynutil.delete('"') + + evening_suffixes = pynini.string_file(get_abs_path("data/time/evening_suffix.tsv")) + evening_suffixes = insert_space + pynutil.delete('suffix: "') + evening_suffixes + pynutil.delete('"') + + night_suffixes = pynini.string_file(get_abs_path("data/time/night_suffix.tsv")) + night_suffixes = insert_space + pynutil.delete('suffix: "') + night_suffixes + pynutil.delete('"') + + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete('"') + ) + + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete('"') + ) + + prefix = ( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.insert("-") + + pynutil.delete('"') + ) + + optional_prefix = pynini.closure(prefix + delete_zero_or_one_space, 0, 1) + optional_suffix = pynini.closure(delete_space + day_suffixes, 0, 1) + graph = hour + delete_space + pynutil.insert(":") + minute + optional_suffix + + for hour_to, suffix in zip( + [hour_to_noon, hour_to_evening, hour_to_night], + [noon_suffixes, evening_suffixes, night_suffixes], + ): + graph |= hour @ hour_to + delete_space + pynutil.insert(":") + minute + delete_space + suffix + + graph |= optional_prefix + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize.py new file mode 100644 index 000000000..0223259db --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize.py @@ -0,0 +1,54 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars in Hebrew. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal_graph = OrdinalFst().fst + + decimal = DecimalFst() + decimal_graph = decimal.fst + + measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst + + time_graph = TimeFst().fst + + date_graph = DateFst().fst + + whitelist_graph = WhiteListFst().fst + + graph = ( + time_graph | date_graph | measure_graph | ordinal_graph | decimal_graph | cardinal_graph | whitelist_graph + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize_final.py new file mode 100644 index 000000000..611181df4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/verbalize_final.py @@ -0,0 +1,44 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.he.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import delete_extra_space, delete_space + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence in Hebrew + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/whitelist.py new file mode 100644 index 000000000..0607e0b37 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/whitelist.py @@ -0,0 +1,50 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import NEMO_ALPHA_HE, GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "mrs." } -> mrs. + """ + + def __init__(self): + super().__init__(name="whitelist", kind="verbalize") + # Keep the prefix if exists and add a dash + optional_prefix = pynini.closure( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_ALPHA_HE, 1) + + pynutil.delete('"') + + delete_space, + 0, + 1, + ) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete('"') + ) + graph = graph @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) + final_graph = optional_prefix + graph + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/he/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/he/verbalizers/word.py new file mode 100644 index 000000000..49c61cf6a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/he/verbalizers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.he.graph_utils import GraphFst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens + e.g. tokens { name: "sleep" } -> sleep + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete('"') + chars + pynutil.delete('"') + graph = char @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/__init__.py new file mode 100644 index 000000000..b650cbece --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.hi.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/address/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/address/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/address/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/address/cities.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/address/cities.tsv new file mode 100644 index 000000000..b49330652 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/address/cities.tsv @@ -0,0 +1,36 @@ +अमरावती +ईटानगर +दिसपुर +पटना +रायपुर +पणजी +गांधीनगर +चंडीगढ़ +शिमला +रांची +बेंगलुरु +तिरुवनंतपुरम +भोपाल +मुंबई +इम्फाल +शिलांग +आइजोल +कोहिमा +भुवनेश्वर +जयपुर +गंगटोक +चेन्नई +हैदराबाद +अगरतला +लखनऊ +देहरादून +कोलकाता +पोर्ट ब्लेयर +दमन +नई दिल्ली +श्रीनगर +जम्मू +लेह +कारगिल +कवरत्ती +पुडुचेरी \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/address/context_cues.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/address/context_cues.tsv new file mode 100644 index 000000000..a40d069b4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/address/context_cues.tsv @@ -0,0 +1,85 @@ +हाउस +प्लॉट +बूथ +अपार्टमेंट +फ्लैट +यूनिट +टावर +कॉम्प्लेक्स +मंजिल +फ्लोर +ब्लॉक +सेक्टर +फेज +रोड +सड़क +मार्ग +स्ट्रीट +गली +राजमार्ग +ड्राइव +डिस्ट्रिक्ट +बाईपास +हाइवे +पार्कवे +कॉलोनी +नगर +पार्क +एस्टेट +बोलवार्ड +मार्केट +सेंटर +पिन +गांव +पास +ब्रिगेड +नियर +स्क्वेर +मॉल +टॉवर +इंस्टीट्यूट +पिलर +मेट्रो +एवेन्यू +वेस्ट +सामने +पीछे +वीया +टाउन +लेन +चौक +चौराहा +बस्ती +मोहल्ला +विहार +एन्क्लेव +पुरम +शहर +जिला +ईस्ट +नॉर्थ +साउथ +प्लाज़ा +ब्रिज +स्टेशन +आर डी +अन्तर्गत +शॉप +क्रॉस +विलेज +हिल +असम +गोवा +महाराष्ट्र +दिल्ली +बैंगलोर +चेन्नई +मुंबई +कोलकाता +हैदराबाद +अहमदाबाद +पुणे +जयपुर +चंडीगढ़ +लखनऊ +इलाहाबाद diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/address/digit_passthrough.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/address/digit_passthrough.tsv new file mode 100644 index 000000000..42633c28b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/address/digit_passthrough.tsv @@ -0,0 +1,10 @@ +۰ ۰ +۱ ۱ +۲ ۲ +۳ ۳ +۴ ۴ +۵ ۵ +۶ ۶ +۷ ۷ +۸ ۸ +۹ ۹ diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/address/ordinals.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/address/ordinals.tsv new file mode 100644 index 000000000..1460d2b3d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/address/ordinals.tsv @@ -0,0 +1,167 @@ +फ़र्स्ट १st +सेकंड २nd +थर्ड ३rd +फ़ोर्थ ४th +फ़िफ्थ ५th +सिक्स्थ ६th +सेवंथ ७th +एटथ ८th +नाइंथ ९th +टेंथ १०th +इलेवंथ ११th +ट्वेल्फ्थ १२th +थर्टींथ १३th +फोर्टींथ १४th +फिफ्टींथ १५th +पाँचवां ५वां +पाँचवीं ५वीं +छठवां ६ठवां +छठवीं ६ठवीं +सातवां ७वां +सातवीं ७वीं +आठवां ८वां +आठवीं ८वीं +नौवां ९वां +नौवीं ९वीं +दसवां १०वां +दसवीं १०वीं +ग्यारहवां ११वां +ग्यारहवीं ११वीं +बारहवां १२वां +बारहवीं १२वीं +तेरहवां १३वां +तेरहवीं १३वीं +चौदहवां १४वां +चौदहवीं १४वीं +पंद्रहवां १५वां +पंद्रहवीं १५वीं +सोलहवां १६वां +सोलहवीं १६वीं +सत्रहवां १७वां +सत्रहवीं १७वीं +अठारहवां १८वां +अठारहवीं १८वीं +उन्नीसवां १९वां +उन्नीसवीं १९वीं +बीसवां २०वां +बीसवीं २०वीं +इक्कीसवां २१वां +इक्कीसवीं २१वीं +बाईसवां २२वां +बाईसवीं २२वीं +तेईसवां २३वां +तेईसवीं २३वीं +चौबीसवां २४वां +चौबीसवीं २४वीं +पच्चीसवां २५वां +पच्चीसवीं २५वीं +छब्बीसवां २६वां +छब्बीसवीं २६वीं +सत्ताईसवां २७वां +सत्ताईसवीं २७वीं +अट्ठाईसवां २८वां +अट्ठाईसवीं २८वीं +उनतीसवां २९वां +उनतीसवीं २९वीं +तीसवां ३०वां +तीसवीं ३०वीं +इकतीसवां ३१वां +इकतीसवीं ३१वीं +बत्तीसवां ३२वां +बत्तीसवीं ३२वीं +तैंतीसवां ३३वां +तैंतीसवीं ३३वीं +चौंतीसवां ३४वां +चौंतीसवीं ३४वीं +पैंतीसवां ३५वां +पैंतीसवीं ३५वीं +छत्तीसवां ३६वां +छत्तीसवीं ३६वीं +सैंतीसवां ३७वां +सैंतीसवीं ३७वीं +अड़तीसवां ३८वां +अड़तीसवीं ३८वीं +उनतालीसवां ३९वां +उनतालीसवीं ३९वीं +चालीसवां ४०वां +चालीसवीं ४०वीं +इकतालीसवां ४१वां +इकतालीसवीं ४१वीं +बयालीसवां ४२वां +बयालीसवीं ४२वीं +तैंतालीसवां ४३वां +तैंतालीसवीं ४३वीं +चवालीसवां ४४वां +चवालीसवीं ४४वीं +पैंतालीसवां ४५वां +पैंतालीसवीं ४५वीं +छियालीसवां ४६वां +छियालीसवीं ४६वीं +सैंतालीसवां ४७वां +सैंतालीसवीं ४७वीं +अड़तालीसवां ४८वां +अड़तालीसवीं ४८वीं +उनचासवां ४९वां +उनचासवीं ४९वीं +पचासवां ५०वां +पचासवीं ५०वीं +इक्यावनवां ५१वां +इक्यावनवीं ५१वीं +बावनवां ५२वां +बावनवीं ५२वीं +तिरपनवां ५३वां +तिरपनवीं ५३वीं +चौवनवां ५४वां +चौवनवीं ५४वीं +पचपनवां ५५वां +पचपनवीं ५५वीं +छप्पनवां ५६वां +छप्पनवीं ५६वीं +सत्तावनवां ५७वां +सत्तावनवीं ५७वीं +अट्ठावनवां ५८वां +अट्ठावनवीं ५८वीं +उनसठवां ५९वां +उनसठवीं ५९वीं +साठवां ६०वां +साठवीं ६०वीं +इकसठवां ६१वां +इकसठवीं ६१वीं +बासठवां ६२वां +बासठवीं ६२वीं +तिरसठवां ६३वां +तिरसठवीं ६३वीं +चौंसठवां ६४वां +चौंसठवीं ६४वीं +पैंसठवां ६५वां +पैंसठवीं ६५वीं +छियासठवां ६६वां +छियासठवीं ६६वीं +सड़सठवां ६७वां +सड़सठवीं ६७वीं +अड़सठवां ६८वां +अड़सठवीं ६८वीं +उनहत्तरवां ६९वां +उनहत्तरवीं ६९वीं +सत्तरवां ७०वां +सत्तरवीं ७०वीं +इकहत्तरवां ७१वां +इकहत्तरवीं ७१वीं +बहत्तरवां ७२वां +बहत्तरवीं ७२वीं +तिहत्तरवां ७३वां +तिहत्तरवीं ७३वीं +चौहत्तरवां ७४वां +चौहत्तरवीं ७४वीं +पचहत्तरवां ७५वां +पचहत्तरवीं ७५वीं +छिहत्तरवां ७६वां +छिहत्तरवीं ७६वीं +सतहत्तरवां ७७वां +सतहत्तरवीं ७७वीं +अठहत्तरवां ७८वां +अठहत्तरवीं ७८वीं +उनासीवां ७९वां +उनासीवीं ७९वीं +अस्सीवां ८०वां +अस्सीवीं ८०वीं diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/address/special_characters.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/address/special_characters.tsv new file mode 100644 index 000000000..bcef2cfdd --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/address/special_characters.tsv @@ -0,0 +1,2 @@ +हाइफ़न - +बटा / \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/address/states.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/address/states.tsv new file mode 100644 index 000000000..531e70da4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/address/states.tsv @@ -0,0 +1,36 @@ +आंध्र प्रदेश +अरुणाचल प्रदेश +असम +बिहार +छत्तीसगढ़ +गोवा +गुजरात +हरियाणा +हिमाचल प्रदेश +झारखंड +कर्नाटक +केरल +मध्य प्रदेश +महाराष्ट्र +मणिपुर +मेघालय +मिज़ोरम +नागालैंड +ओडिशा +पंजाब +राजस्थान +सिक्किम +तमिलनाडु +तेलंगाना +त्रिपुरा +उत्तर प्रदेश +उत्तराखंड +पश्चिम बंगाल +अंडमान और निकोबार द्वीप समूह +चंडीगढ़ +दादरा और नगर हवेली और दमन और दीव +दिल्ली +जम्मू और कश्मीर +लद्दाख +लक्षद्वीप +पुडुचेरी \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/date/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/date/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/date/century.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/date/century.tsv new file mode 100644 index 000000000..9369023e0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/date/century.tsv @@ -0,0 +1,5 @@ +ई.पू. ईसा पूर्व +ई. ईस्वी +ई. ईसवी +वर्ष पूर्व वर्ष पूर्व +शताब्दी शताब्दी \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/date/months.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/date/months.tsv new file mode 100644 index 000000000..0913a5909 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/date/months.tsv @@ -0,0 +1,14 @@ +जनवरी +फ़रवरी +फरवरी +मार्च +अप्रैल +अप्रील +मई +जून +जुलाई +अगस्त +सितंबर +अक्टूबर +नवंबर +दिसंबर \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/measure/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/measure/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/measure/measurements.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/measure/measurements.tsv new file mode 100644 index 000000000..d472e15df --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/measure/measurements.tsv @@ -0,0 +1,295 @@ +°C सेल्सियस +°C सेंटीग्रेड +°C डिग्री सेल्सियस +°C डिग्री सेंटीग्रेड +°F फारेनहाइट +°F डिग्री फारेनहाइट +°F फेरनहाइट +°F फॉरेनहाइट +°F डिग्री फेरनहाइट +K कैल्विन +K केल्विन +g ग्राम +kg किलोग्राम +kg किलो ग्राम +kg किलो +kg किग्रा +mg मिलिग्राम +mg मिलीग्राम +mg मिली ग्राम +mg मिलि ग्राम +cg सेंटीग्राम +cg सेंटी ग्राम +dg डेसिग्राम +dg डेसि ग्राम +Tg टेराग्राम +Tg टेरा ग्राम +Mg मेगाग्राम +Mg मेगा ग्राम +Gg गीगाग्राम +Gg गीगा ग्राम +hg हेक्टोग्राम +hg हेक्टो ग्राम +dag डेकाग्राम +dag डेका ग्राम +lb पाउंड +oz आउन्स +t टन +t मीट्रिक टन +t शिपिंग टन +st स्टोन +oz औंस +q क्विंटल +q क्विटंल +m मीटर +cm सेंटीमीटर +cm सेंटी मीटर +cm सेन्टीमीटर +cm सेन्टी मीटर +mm मिलीमीटर +mm मिली मीटर +km किलोमीटर +km किलो मीटर +dm डेसिमीटर +dm डेसि मीटर +dam डेकामीटर +dam डेका मीटर +nm नैनोमीटर +nm नैनो मीटर +hm हेक्टोमीटर +hm हेक्टो मीटर +Mm मेगामीटर +Mm मेगा मीटर +my मिरिया मीटर +my मिरियामीटर +mi मील +in इंच +ft फुट +ft फीट +ft फिट +in इंच +yd यार्ड +µm माइक्रोमीटर +µm माइक्रो मीटर +µm माइक्रोन +µm माइक्रॉन +m² वर्ग मीटर +m² वर्गमीटर +cm² वर्ग सेंटीमीटर +cm² वर्गसेंटीमीटर +mm² वर्ग मिलीमीटर +mm² वर्गमिलीमीटर +km² वर्ग किलोमीटर +km² वर्गकिलोमीटर +hm² वर्गहेक्टोमीटर +hm² वर्ग हेक्टो मीटर +hm² वर्ग हेक्टोमीटर +dm² वर्गडेसिमीटर +dm² वर्ग डेसि मीटर +dm² वर्ग डेसिमीटर +dm² वर्गडेसीमीटर +dm² वर्ग डेसी मीटर +dm² वर्ग डेसीमीटर +dam² वर्ग डेका मीटर +dam² वर्ग डेकामीटर +dam² वर्गडेकामीटर +yd² वर्गयार्ड +yd² वर्ग यार्ड +ft² वर्ग फुट +ft² वर्गफुट +ft² वर्ग फीट +ft² वर्गफीट +ac² वर्ग एकड़ +ac² वर्गएकड़ +my² वर्ग मिरिया मीटर +my² वर्गमिरियामीटर +my² वर्ग मिरियामीटर +mi² वर्ग मील +mi² वर्गमील +nm² वर्गनैनोमीटर +nm² वर्ग नैनो मीटर +nm² वर्ग नैनोमीटर +µm² वर्ग माइक्रोमीटर +µm² वर्ग माइक्रो मीटर +µm² वर्गमाइक्रोमीटर +in² वर्ग इंच +in² वर्गइंच +my³ घन मिरिया मीटर +my³ घनमिरियामीटर +my³ घन मिरियामीटर +mi³ घन मील +mi³ घनमील +ac³ घन एकड़ +ac³ घनएकड़ +ha हेक्टेयर +ac एकड़ +गज गज +गज² वर्ग गज +गज² वर्गगज +गज³ घन गज +गज³ घनगज +m³ क्यूबिक मीटर +m³ क्यूबिकमीटर +m³ घन मीटर +m³ घनमीटर +cm³ क्यूबिक सेंटीमीटर +cm³ घन सेंटीमीटर +cm³ घनसेंटीमीटर +cm³ क्यूबिकसेंटीमीटर +mm³ क्यूबिक मिलीमीटर +mm³ घन मिलीमीटर +mm³ घनमिलीमीटर +mm³ क्यूबिकमिलीमीटर +dam³ क्यूबिकडेकामीटर +dam³ क्यूबिक डेका मीटर +dam³ घन डेका मीटर +dam³ घन डेकामीटर +dam³ घनडेकामीटर +µm³ घन माइक्रोमीटर +µm³ घन माइक्रो मीटर +µm³ घनमाइक्रोमीटर +mL³ घन मिली लीटर +mL³ घनमिलीलीटर +mL³ घन मिलीलीटर +L लीटर +kL किलोलीटर +kL किलो लीटर +mL मिलीलीटर +mL मिली लीटर +mL² वर्ग मिली लीटर +mL² वर्गमिलीलीटर +mL² वर्ग मिलीलीटर +dL डेसीलीटर +dL डेसी लीटर +hL हेक्टोलीटर +hL हेक्टो लीटर +cL सेंटीलीटर +daL डेकालीटर +daL डेका लीटर +dL² वर्ग डेसीलीटर +dL² वर्गडेसीलीटर +dL² वर्ग डेसी लीटर +daL² वर्ग डेकालीटर +daL² वर्गडेकालीटर +daL² वर्ग डेका लीटर +dL³ घन डेसीलीटर +dL³ घनडेसीलीटर +dL³ घन डेसी लीटर +daL³ घन डेकालीटर +daL³ घनडेकालीटर +daL³ घन डेका लीटर +GB गीगाबाइट +in³ घन इंच +in³ घनइंच +ft³ घन फुट +ft³ घनफुट +ft³ घन फीट +ft³ घनफीट +yd³ घन यार्ड +yd³ घनयार्ड +my³ घन मिरिया मीटर +my³ घनमिरियामीटर +my³ घन मिरियामीटर +mi³ घन मील +mi³ घनमील +dm³ घनडेसिमीटर +dm³ घन डेसि मीटर +dm³ घन डेसिमीटर +dm³ घनडेसीमीटर +dm³ घन डेसी मीटर +dm³ घन डेसीमीटर +km³ घन किलोमीटर +km³ घनकिलोमीटर +nm³ घननैनोमीटर +nm³ घन नैनो मीटर +nm³ घन नैनोमीटर +mm³ घन मिलीमीटर +mm³ घनमिलीमीटर +qt क्वार्ट +gal गैलन +pt पिंट +pt पाइंट +W वाट +MW मेगावाट +MW मेगा वाट +KW किलोवाट +KW किलो वाट +b बिट +Mb मेगा बिट +Mb मेगाबिट +B बाइट +GB गीगाबाइट +KB किलोबाइट +TB टेराबाइट +MB मेगाबाइट +MB मैगाबाइट +PB पेटाबाइट +EB एक्साबाइट +ZB जेट्टाबाइट +ZB जेटाबाइट +YB योट्टाबाइट +YB योटाबाइट +BB ब्रोन्ट्टोबाइट +BB ब्रोन्टोबाइट +C कूलंब +V वोल्ट +Pa पास्कल +A एम्पीयर +A ऐंपीयर +J जूल +s सेकंड +s सेकेंड +h घंटा +h घंटे +h घंटों +m मिनट +ha हेक्टेयर +ha² वर्ग हेक्टेयर +ha² वर्गहेक्टेयर +Ω ओम +MΩ मेगओम +MΩ मेग ओम +MΩ मेगाओम +MΩ मेगा ओम +doz दर्जन +Hz फ्रीक्वेंसी +Hz हर्ट्ज +GHz गीगाहर्ट्ज़ +KHz किलोहर्ट्ज़ +N न्यूटन +dB डेसीबल +yr साल +yr सालों +yr वर्ष +yr वर्षों +hp हॉर्स पावर +hp हॉर्सपॉवर +hp हॉर्सपावर +hp अश्वशक्ति +hp अश्वशक्त +d दिनों +d दिन +mo महीने +mo महीना +ct कैरेट +pH पी एच +pH पीएच +km/h किलोमीटर प्रति घंटा +km/h किलोमीटर प्रति घंटे +km/m किलोमीटर प्रति मिनट +km/m किलोमीटर प्रति मिनट +m/hr मीटर प्रति घंटा +m/hr मीटर प्रति घंटे +mi/s मील प्रति सेकंड +mi/s मील प्रति सेकेंड +mi/h मील प्रति घंटा +mi/h मील प्रति घंटे +mi/m मील प्रति मिनट +₹/ac रुपए प्रति एकड़ +w हफ़्ते +w हफ़्ता +w सप्ताह +सदियां सदियां +सदियाँ सदियाँ +सदियों सदियों +सदी सदी diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/money/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/money/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/money/currency.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/money/currency.tsv new file mode 100644 index 000000000..3ee478688 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/money/currency.tsv @@ -0,0 +1,181 @@ +؋ अफगान अफगानी +l अल्बानियन लेक +دج अल्जीरियाई दिनार +kz अंगोलीन कुआनज़ा +aed अरब अमीरात दिरहम +$ अर्जेंटीनी पेसो +֏ आर्मेनियाई ड्राम +ƒ अरूबान फ्लोरिन +a$ ऑस्ट्रेलियाई डॉलर +₼ अजरबैजानी मनात +b$ बहामियन डॉलर +.د.ب बहरीन दिरहम +৳ बांग्लादेशी टका +$ बारबेडियन डॉलर +br बेलारूसी रूबल +bz$ बेलीज डॉलर +$ बर्मूडियन डॉलर +nu भूटानी न्गुलट्रुम +₿ बिटकॉइन +bs बोलिवियन बोलिवियानो +km बोस्निया और हर्जेगोविना का मार्क +p बोत्सवाना पुला +r$ ब्राजीलियाई रियाल +£ ब्रिटिश पाउंड +b$ ब्रुनेई डॉलर +лв बुल्गारियाई लेव +fbu बुरुंडी फ्रैंक +៛ कंबोडियन रियल +ca$ कनाडाई डॉलर +cve केप वर्डे एस्कूडो +cve केप वर्डियन एस्कूडो +ci$ कैमन आइलैंड्स डॉलर +cfa सीएफए फ्रैंक +$ चिली पेसो +元 चीनी युआन +$ कोलंबियन पेसो +cf कोमोरियन फ्रैंक +fc कांगोलेस फ्रैंक +₡ कोस्टा रिका कोलन +kn क्रोएशियन कुना +cuc$ क्यूबन पेसो +cyp साइप्रस पाउंड +kč चेक कोरोना +d डालासी +kr डेनिश क्रोन +fdj जिबूती फ्रैंक +$ डॉलर +rd$ डोमिनिकन पेसो +$ पूर्व कैरेबियाई डॉलर +e£ मिस्री पाउंड +aed अमीराती दिरहम +nkf इरिट्रियन नक्फा +Ξ एथेरियम +br इथियोपियाई बिर्क +€ यूरो +fk£ फाल्कलैंड आइलैंड्स पाउंड +fim फिनिश मार्क्का +₾ जॉर्जियन लारी +dm जर्मन मार्क +gh₵ घानियन सेडी +q ग्वाटेमालान केट्ज़ल +fg गिनी फ्रैंक +g$ गुयानीज़ डॉलर +g हैतीयन गॉर्ड +l होंडुरन लेम्पिरा +hk$ हांगकांग डॉलर +ft हंगेरियन फोरिंट +íkr आइसलैंडिक क्रोना +₹ भारतीय रुपया +₹ रुपया +₹ रुपिया +₹ रुपेया +₹ रुपये +₹ रुपए +p पैसा +p पैसे +rp इंडोनेशियाई रुपिया +﷼ ईरानी रियाल +ع.د इराकी दिनार +₪ इजरायली नया शेकेल +₪ इजरायली शेकेल +j$ जमैकी डॉलर +¥ जापानी येन +ينار जॉर्डनियन दिनार +₸ कजाखस्तानी टेंगे +ksh केन्याई शिलिंग +ك कुवैती दिनार +k क्यात +som किर्गिज़स्तानी सॉम +₭ लाओ किप +ლარი लारी +ل.ل लेबनानी पाउंड +l लेसोथो लोति +ld$ लाइबेरियन डॉलर +ld लाइबियाई दिनार +l लिलांगेनी +ł लाइटकॉइन +mop$ मकाउ पाटाका +ar मलागासी एरियारी +k मलावियन क्वाचा +rm मलेशियन रिंगगिट +rf मालदीवियन रूफिया +mro मोरिटानियन औगुइया +₨ मॉरिटियन रुपया +$ मेक्सिकन पेसो +l मोल्डोवन ल्यू +ɱ मोनेरो +₮ मंगोलियन टोग्रोग +dh मोरक्कन दिरहम +mt मोज़ाम्बिकन मेटिकल +n$ नामीबियन डॉलर +rs नेपाली रुपया +ƒ नीदरलैंड्स एंटिलियन गिल्डर +nt$ नए ताइवान डॉलर +nz$ नए ज़ीलैंड डॉलर +c$ निकारागुआन कॉर्डोबा +₦ नाइजीरियन नायरा +₩ उत्तर कोरियाई वोन +kr नॉर्वेजियन क्रोन +ر.ع ओमानी रियाल +um औगुइया +pkr पाकिस्तानी रुपया +b/. पनामाई बाल्बोआ +₲ पराग्वेगियन गुआरानी +s/. पेरूवियन सोल +₱ फिलीपीन पेसो +zł पोलिश ज़्लॉटी +£ पाउंड स्टर्लिंग +ر.ق क़ातरी रियाल +£ क्विड +r रियाल +xrp रिपल्स +lei रोमानियाई ल्यू +₹ रुपया +₽ रूसी रूबल +r₣ रवांडा फ्रैंक +shp सेंट हेलेना पाउंड्स +db साओ टोमे और प्रिंसिपे दोबरा +sar सऊदी रियाल +ден दूसरा मैसेडोनियन दिनार +rsd सर्बियाई दिनार +scr सेशेल्स रुपया +le सिएरा लियोनेयाई लियोन +s$ सिंगापुर डॉलर +sh.so. सोमाली शिलिंग +tjs सोमोनी +r दक्षिण अफ्रीकी रैंड +₩ दक्षिण कोरियाई वोन +ss£ दक्षिण सूडानी पाउंड +රු श्रीलंकाई रुपया +sdg सूडानी पाउंड +sr$ सुरिनामी डॉलर +kr स्वीडिश क्रोना +chf स्विस फ्रैंक +£s सीरियाई पाउंड +৳ टका +৳ टके +tzs तंजानियाई शिलिंग +₮ टेथर +฿ थाई बात +tt$ त्रिनिदाद और टोबैगो डॉलर +د.ت ट्यूनीशियन दिनार +₺ तुर्की लिरा +m तुर्कमेन नए मनात +ush युगांडा शिलिंग +₴ यूक्रेनी ग्रिव्ना +$ संयुक्त राज्य डॉलर +$u उरुग्वयन पेसो +$ अमेरिकी डॉलर +$ डॉलर +som उज्बेकिस्तान सॉम +bs. वेनेजुएलन बोलिवार +₫ वियतनामी डोंग +₩ वोन +₩ वॉन +﷼ यमनी रियाल +¥ येन +₺ लीरा +₦ नाइरा +¢ सेंट्स +¢ सेंट diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/digit.tsv new file mode 100644 index 000000000..c700ab475 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/digit.tsv @@ -0,0 +1,13 @@ +१ एक +२ दो +३ तीन +४ चार +५ पाँच +५ पांच +६ छः +६ छ: +६ छह +६ छे +७ सात +८ आठ +९ नौ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/hundred.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/hundred.tsv new file mode 100644 index 000000000..d31a29ccb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/hundred.tsv @@ -0,0 +1 @@ +सौ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/labels_exception.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/labels_exception.tsv new file mode 100644 index 000000000..f5c52bcfb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/labels_exception.tsv @@ -0,0 +1,4 @@ +शून्य +एक +दो +तीन diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/paune.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/paune.tsv new file mode 100644 index 000000000..f56b256e6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/paune.tsv @@ -0,0 +1,231 @@ +१ दो +२ तीन +३ चार +४ पाँच +४ पांच +५ छः +५ छह +५ छे +६ सात +७ आठ +८ नौ +९ दस +१० ग्यारह +११ बारह +१२ तेरह +१३ चौदह +१४ पन्द्रह +१४ पंद्रह +१५ सोलह +१६ सत्रह +१६ सतरह +१७ अठारह +१७ अट्ठारह +१८ उन्नीस +१८ उनीस +१९ बीस +२० इक्कीस +२० इकीस +२० ईकीस +२१ बाईस +२१ बाइस +२२ तेईस +२२ तेइस +२३ चौबीस +२४ पच्चीस +२४ पचीस +२५ छब्बीस +२५ छबीस +२६ सत्ताईस +२६ सत्ताइस +२६ सताईस +२६ सताइस +२७ अट्ठाईस +२७ अट्ठाइस +२७ अठाईस +२७ अठाइस +२८ उनतीस +२८ उन्तीस +२९ तीस +३० इकतीस +३० इकतिस +३० इकत्तीस +३० इकत्तिस +३१ बत्तीस +३१ बत्तिस +३१ बतीस +३१ बतिस +३२ तैंतीस +३२ तैंतिस +३२ तैंत्तीस +३२ तैंत्तिस +३२ तेतीस +३२ तेंतीस +३३ चौंतीस +३३ चौंतिस +३३ चौंत्तीस +३३ चौंत्तिस +३४ पैंतीस +३४ पैंतिस +३४ पैंत्तीस +३४ पैंत्तिस +३५ छत्तीस +३५ छत्तिस +३५ छतीस +३५ छतिस +३६ सैंतीस +३६ सैंतिस +३६ सैंत्तीस +३६ सैंत्तिस +३७ अड़तीस +३७ अड़तिस +३७ अड़त्तीस +३७ अड़त्तिस +३८ उनतालीस +३८ उनतालिस +३८ उनत्तालीस +३८ उनत्तालिस +३८ उन्तालीस +३८ उन्तालिस +३९ चालीस +४० इकतालीस +४० इकतालिस +४० इक्तालीस +४१ बयालीस +४१ बयालिस +४१ ब्यालीस +४२ तैंतालीस +४२ तैंतालिस +४३ चौवालीस +४३ चौवालिस +४३ चवालीस +४३ चवालिस +४३ चौंतालीस +४४ पैंतालीस +४४ पैंतालिस +४५ छियालीस +४५ छियालिस +४५ छयालीस +४६ सैंतालीस +४६ सैंतालिस +४६ सैतालिस +४७ अड़तालीस +४७ अड़तालिस +४८ उनचास +४९ पचास +५० इक्यावन +५० इकयावन +५१ बावन +५२ तिरपन +५२ तिरेपन +५३ चौवन +५४ पचपन +५५ छप्पन +५५ छपन +५६ सत्तावन +५६ सतावन +५७ अट्ठावन +५७ अठावन +५८ उनसठ +५८ उनसठ +५९ साठ +६० इकसठ +६१ बासठ +६१ बासट +६२ तिरसठ +६२ तिरेसठ +६३ चौंसठ +६४ पैंसठ +६५ छियासठ +६५ छयासठ +६६ सड़सठ +६७ अड़सठ +६८ उनहत्तर +६८ उनहतर +६९ सत्तर +६९ सतर +७० इकहत्तर +७० इकहतर +७० इक्हत्तर +७० इकत्तर +७१ बहत्तर +७१ बहतर +७२ तिहत्तर +७२ तिहतर +७३ चौहत्तर +७३ चौहतर +७४ पचहत्तर +७४ पचहतर +७४ पिछत्तर +७४ पिछतर +७५ छिहत्तर +७५ छिहतर +७५ छियत्तर +७६ सतहत्तर +७६ सतहतर +७६ सतत्तर +७७ अठहत्तर +७७ अठहतर +७८ उन्यासी +७८ उन्यासि +७८ उनासी +७८ उनासि +७९ अस्सी +७९ अस्सि +८० इक्यासी +८० इक्यासि +८१ बयासी +८१ बयासि +८१ ब्यासी +८१ ब्यासि +८१ बिरासी +८२ तिरासी +८२ तिरासि +८२ तेरासी +८३ चौरासी +८३ चौरासि +८४ पचासी +८४ पचासि +८४ पिचयासी +८४ पिचयासि +८४ पिचासी +८५ छियासी +८५ छियासि +८६ सत्तासी +८६ सत्तासि +८६ सतासी +८६ सतासि +८७ अट्ठासी +८७ अट्ठासि +८७ अठासी +८७ अठासि +८८ नवासी +८८ नवासि +८९ नब्बे +९० इक्यानबे +९० इक्यानवे +९१ बानबे +९१ बानवे +९२ तिरानबे +९२ तिरानवे +९३ चौरानबे +९३ चौरानवे +९४ पंचानबे +९४ पंचानवे +९४ पचानवे +९४ पिचयानवे +९४ पिचयानबे +९४ पिच्यानवे +९४ पिच्यानबे +९५ छियानबे +९५ छियानवे +९६ सत्तानबे +९६ सत्तानवे +९७ अट्ठानबे +९७ अट्ठानवे +९७ अठानवे +९७ अठानबे +९८ निन्यान्बे +९८ निन्यानबे +९८ निन्यानवे +९८ निन्यान्वे diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/teens_and_ties.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..3968a1320 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/teens_and_ties.tsv @@ -0,0 +1,220 @@ +१० दस +११ ग्यारह +१२ बारह +१३ तेरह +१४ चौदह +१५ पन्द्रह +१५ पंद्रह +१६ सोलह +१७ सत्रह +१७ सतरह +१८ अठारह +१८ अठाहर +१८ अट्ठारह +१९ उन्नीस +१९ उनीस +२० बीस +२१ इक्कीस +२१ इकीस +२१ ईकीस +२२ बाईस +२२ बाइस +२३ तेईस +२३ तेइस +२४ चौबीस +२५ पच्चीस +२५ पचीस +२६ छब्बीस +२६ छबीस +२७ सत्ताईस +२७ सत्ताइस +२७ सताईस +२७ सताइस +२८ अट्ठाईस +२८ अट्ठाइस +२८ अठाईस +२८ अठाइस +२९ उनतीस +२९ उन्तीस +३० तीस +३१ इकतीस +३१ इकतिस +३१ इकत्तीस +३१ इकत्तिस +३२ बत्तीस +३२ बत्तिस +३२ बतीस +३२ बतिस +३३ तैंतीस +३३ तैंतिस +३३ तैंत्तीस +३३ तैंत्तिस +३३ तेतीस +३३ तेंतीस +३४ चौंतीस +३४ चौंतिस +३४ चौंत्तीस +३४ चौंत्तिस +३५ पैंतीस +३५ पैंतिस +३५ पैंत्तीस +३५ पैंत्तिस +३६ छत्तीस +३६ छत्तिस +३६ छतीस +३६ छतिस +३७ सैंतीस +३७ सैंतिस +३७ सैंत्तीस +३७ सैंत्तिस +३८ अड़तीस +३८ अड़तिस +३८ अड़त्तीस +३८ अड़त्तिस +३९ उनतालीस +३९ उनतालिस +३९ उनत्तालीस +३९ उनत्तालिस +३९ उन्तालीस +३९ उन्तालिस +४० चालीस +४१ इकतालीस +४१ इकतालिस +४१ इक्तालीस +४२ बयालीस +४२ बयालिस +४२ ब्यालीस +४३ तैंतालीस +४३ तैंतालिस +४४ चौवालीस +४४ चौवालिस +४४ चवालीस +४४ चवालिस +४४ चौंतालीस +४५ पैंतालीस +४५ पैंतालिस +४६ छियालीस +४६ छियालिस +४६ छयालीस +४७ सैंतालीस +४७ सैंतालिस +४७ सैतालिस +४८ अड़तालीस +४८ अड़तालिस +४९ उनचास +५० पचास +५१ इक्यावन +५१ इकयावन +५२ बावन +५३ तिरपन +५३ तिरेपन +५४ चौवन +५५ पचपन +५६ छप्पन +५६ छपन +५७ सत्तावन +५७ सतावन +५८ अट्ठावन +५८ अठावन +५९ उनसठ +६० साठ +६१ इकसठ +६२ बासठ +६२ बासट +६३ तिरसठ +६३ तिरेसठ +६४ चौंसठ +६५ पैंसठ +६६ छियासठ +६६ छयासठ +६७ सड़सठ +६८ अड़सठ +६९ उनहत्तर +६९ उनहतर +७० सत्तर +७० सतर +७१ इकहत्तर +७१ इकहतर +७१ इक्हत्तर +७१ इकत्तर +७२ बहत्तर +७२ बहतर +७३ तिहत्तर +७३ तिहतर +७४ चौहत्तर +७४ चौहतर +७५ पचहत्तर +७५ पचहतर +७५ पिछत्तर +७५ पिछतर +७६ छिहत्तर +७६ छिहतर +७६ छियत्तर +७७ सतहत्तर +७७ सतहतर +७७ सतत्तर +७८ अठहत्तर +७८ अठहतर +७९ उन्यासी +७९ उन्यासि +७९ उनासी +७९ उनासि +८० अस्सी +८० अस्सि +८१ इक्यासी +८१ इक्यासि +८२ बयासी +८२ बयासि +८२ ब्यासी +८२ ब्यासि +८२ बिरासी +८३ तिरासी +८३ तिरासि +८३ तेरासी +८४ चौरासी +८४ चौरासि +८५ पचासी +८५ पचासि +८५ पिचयासी +८५ पिचयासि +८५ पिचासी +८६ छियासी +८६ छियासि +८७ सत्तासी +८७ सत्तासि +८७ सतासी +८७ सतासि +८८ अट्ठासी +८८ अट्ठासि +८८ अठासी +८८ अठासि +८९ नवासी +८९ नवासि +९० नब्बे +९१ इक्यानबे +९१ इक्यानवे +९२ बानबे +९२ बानवे +९३ तिरानबे +९३ तिरानवे +९४ चौरानबे +९४ चौरानवे +९५ पंचानबे +९५ पंचानवे +९५ पचानवे +९५ पिचयानवे +९५ पिचयानबे +९५ पिच्यानवे +९५ पिच्यानबे +९६ छियानबे +९६ छियानवे +९७ सत्तानबे +९७ सत्तानवे +९८ अट्ठानबे +९८ अट्ठानवे +९८ अठानवे +९८ अठानबे +९९ निन्यान्बे +९९ निन्यानबे +९९ निन्यानवे +९९ निन्यान्वे diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/thousands.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/thousands.tsv new file mode 100644 index 000000000..7a5306053 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/thousands.tsv @@ -0,0 +1,8 @@ +हज़ार +लाख +करोड़ +अरब +खरब +नील +पद्म +शंख \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/zero.tsv new file mode 100644 index 000000000..727ccdf16 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/numbers/zero.tsv @@ -0,0 +1 @@ +० शून्य \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit.tsv new file mode 100644 index 000000000..4c8d7381a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit.tsv @@ -0,0 +1,6 @@ +पहला एक +दूसरा दो +तीसरा तीन +चौथा चार +पाँचवाँ पाँच +छठा छः \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit_fem.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit_fem.tsv new file mode 100644 index 000000000..ce6c25599 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/digit_fem.tsv @@ -0,0 +1,6 @@ +एकवी एक +दोवी दो +तीनवी तीन +चारवी चार +पांचवी पाँच +छठवी छः \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit.tsv new file mode 100644 index 000000000..426a50e22 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit.tsv @@ -0,0 +1,12 @@ +एकवां एक +दोवां दो +तीनवां तीन +चारवां चार +पांचवां पाँच +छठवां छः +एकवाँ एक +दोवाँ दो +तीनवाँ तीन +चारवाँ चार +पांचवाँ पाँच +छठवाँ छः \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit_fem.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit_fem.tsv new file mode 100644 index 000000000..58d11fa79 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/hundred_digit_fem.tsv @@ -0,0 +1,12 @@ +एकवीं एक +दोवीं दो +तीनवीं तीन +चारवीं चार +पाँचवीं पाँच +छहवीं छः +एकवी एक +दोवी दो +तीनवी तीन +चारवी चार +पाँचवी पाँच +छहवी छः \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/morph_features.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/morph_features.tsv new file mode 100644 index 000000000..ca9c21743 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/morph_features.tsv @@ -0,0 +1,6 @@ +वाँ +वां +वीं +वी +वें +वे diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix.tsv new file mode 100644 index 000000000..c94088ab2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix.tsv @@ -0,0 +1,4 @@ +वाँ +वां +वें +वे \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix_fem.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix_fem.tsv new file mode 100644 index 000000000..b52de3a74 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/suffix_fem.tsv @@ -0,0 +1,2 @@ +वीं +वी \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties.tsv new file mode 100644 index 000000000..9fce3aa01 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties.tsv @@ -0,0 +1 @@ +नब्बेवाँ नब्बे \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties_fem.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties_fem.tsv new file mode 100644 index 000000000..ea4c655d5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/ordinals/teens_and_ties_fem.tsv @@ -0,0 +1 @@ +नब्बेवी नब्बे \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/telephone/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/telephone/context_cues.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/context_cues.tsv new file mode 100644 index 000000000..8373c52df --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/context_cues.tsv @@ -0,0 +1,15 @@ +mobile नंबर +mobile मोबाइल +mobile फोन +mobile कॉल +landline नंबर +landline मोबाइल +landline फोन +landline लैंडलाइन +landline कॉल +pincode पिन +pincode कोड +pincode पिनकोड +credit नंबर +credit कार्ड +credit क्रेडिट \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/telephone/country_codes.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/country_codes.tsv new file mode 100644 index 000000000..268d326b1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/country_codes.tsv @@ -0,0 +1 @@ +९१ नौ एक \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_digit.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_digit.tsv new file mode 100644 index 000000000..6049cbf50 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_digit.tsv @@ -0,0 +1,9 @@ +१ one +२ two +३ three +४ four +५ five +६ six +७ seven +८ eight +९ nine \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_zero.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_zero.tsv new file mode 100644 index 000000000..769cbb603 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/eng_zero.tsv @@ -0,0 +1 @@ +० zero \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/telephone/teens_and_ties_eng_to_hin.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/teens_and_ties_eng_to_hin.tsv new file mode 100644 index 000000000..ac37b55f2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/telephone/teens_and_ties_eng_to_hin.tsv @@ -0,0 +1,90 @@ +१० ten +११ eleven +१२ twelve +१३ thirteen +१४ fourteen +१५ fifteen +१६ sixteen +१७ seventeen +१८ eighteen +१९ nineteen +२० twenty +२१ twenty one +२२ twenty two +२३ twenty three +२४ twenty four +२५ twenty five +२६ twenty six +२७ twenty seven +२८ twenty eight +२९ twenty nine +३० thirty +३१ thirty one +३२ thirty two +३३ thirty three +३४ thirty four +३५ thirty five +३६ thirty six +३७ thirty seven +३८ thirty eight +३९ thirty nine +४० forty +४१ forty one +४२ forty two +४३ forty three +४४ forty four +४५ forty five +४६ forty six +४७ forty seven +४८ forty eight +४९ forty nine +५० fifty +५१ fifty one +५२ fifty two +५३ fifty three +५४ fifty four +५५ fifty five +५६ fifty six +५७ fifty seven +५८ fifty eight +५९ fifty nine +६० sixty +६१ sixty one +६२ sixty two +६३ sixty three +६४ sixty four +६५ sixty five +६६ sixty six +६७ sixty seven +६८ sixty eight +६९ sixty nine +७० seventy +७१ seventy one +७२ seventy two +७३ seventy three +७४ seventy four +७५ seventy five +७६ seventy six +७७ seventy seven +७८ seventy eight +७९ seventy nine +८० eighty +८१ eighty one +८२ eighty two +८३ eighty three +८४ eighty four +८५ eighty five +८६ eighty six +८७ eighty seven +८८ eighty eight +८९ eighty nine +९० ninety +९१ ninety one +९२ ninety two +९३ ninety three +९४ ninety four +९५ ninety five +९६ ninety six +९७ ninety seven +९८ ninety eight +९९ ninety nine diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/time/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/time/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/time/hour_for_paune.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/time/hour_for_paune.tsv new file mode 100644 index 000000000..8bb4c67ca --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/time/hour_for_paune.tsv @@ -0,0 +1,15 @@ +१२ एक +१ दो +२ तीन +३ चार +४ पाँच +४ पांच +५ छः +५ छह +५ छे +६ सात +७ आठ +८ नौ +९ दस +१० ग्यारह +११ बारह diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/whitelist/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/whitelist/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/whitelist/whitelist.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/whitelist/whitelist.tsv new file mode 100644 index 000000000..8cfd0e19f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/whitelist/whitelist.tsv @@ -0,0 +1,7 @@ +१/४ पाव +कु. कुमारी +स्मि. श्रीमती +श्री. श्री +श्री. श्रीमान +मा. मास्टर +डॉ. डॉक्टर diff --git a/nemo_text_processing/inverse_text_normalization/hi/graph_utils.py b/nemo_text_processing/inverse_text_normalization/hi/graph_utils.py new file mode 100644 index 000000000..b002efa52 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/graph_utils.py @@ -0,0 +1,204 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + +NEMO_HI_DIGIT = pynini.union("०", "१", "२", "३", "४", "५", "६", "७", "८", "९").optimize() +DEVANAGARI_DIGIT = ["०", "१", "२", "३", "४", "५", "६", "७", "८", "९"] + +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_ZWNJ = u"\u200c" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_CHAR, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("ऋणात्मक", "नकारात्मक").optimize() + + +def integer_to_devanagari(n: int) -> str: + return ''.join(DEVANAGARI_DIGIT[int(d)] for d in str(n)) + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/cardinal.py new file mode 100644 index 000000000..7fcdcf348 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/cardinal.py @@ -0,0 +1,262 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + MINUS, + NEMO_HI_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. ऋण तेईस -> cardinal { integer: "२३" negative: "-" } + + Args: + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__(self, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="cardinal", kind="classify") + self.input_case = input_case + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + graph_teens_and_ties = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")).invert() + graph_paune = pynini.string_file(get_abs_path("data/numbers/paune.tsv")).invert() + self.graph_zero = graph_zero + self.graph_digit = graph_digit + self.graph_single_digit_with_zero = pynutil.insert("०") + graph_digit + self.graph_teens_and_ties = graph_teens_and_ties + self.graph_two_digit = graph_teens_and_ties | (pynutil.insert("०") + graph_digit) + graph_hundred = pynini.cross("सौ", "") + delete_hundred = pynutil.delete("सौ") + delete_thousand = pynutil.delete("हज़ार") | pynutil.delete("हजार") + graph_hundred_component = pynini.union(graph_digit + delete_space + graph_hundred, pynutil.insert("०")) + graph_hundred_component += delete_space + graph_hundred_component += self.graph_two_digit | pynutil.insert("००") + + graph_hundred_component_at_least_one_none_zero_digit = graph_hundred_component @ ( + pynini.closure(NEMO_HI_DIGIT) + (NEMO_HI_DIGIT - "०") + pynini.closure(NEMO_HI_DIGIT) + ) + self.graph_hundred_component_at_least_one_none_zero_digit = ( + graph_hundred_component_at_least_one_none_zero_digit + ) + + # Transducer for eleven hundred -> 1100 or twenty one hundred eleven -> 2111 + graph_hundred_as_thousand = pynini.union( + graph_teens_and_ties + delete_space + graph_hundred, pynutil.insert("०") + ) + graph_hundred_as_thousand += delete_space + graph_hundred_as_thousand += self.graph_two_digit | pynutil.insert("००") + graph_hundred_as_thousand |= pynutil.add_weight( + pynutil.delete("साढ़े") + + delete_space + + graph_digit + + pynutil.insert("५००", weight=-0.1) + + delete_space + + delete_thousand, + -0.1, + ) + graph_hundred_as_thousand |= pynutil.add_weight( + pynutil.delete("सवा") + + delete_space + + graph_digit + + pynutil.insert("२५०", weight=-0.1) + + delete_space + + delete_thousand, + -0.1, + ) + graph_hundred_as_thousand |= pynutil.add_weight( + pynutil.delete("पौने") + + delete_space + + graph_paune + + pynutil.insert("७५०", weight=-0.1) + + delete_space + + delete_thousand, + -0.1, + ) + graph_hundred_as_thousand |= pynutil.add_weight( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + pynutil.insert("१५००", weight=-0.1) + + delete_space + + delete_thousand, + -0.1, + ) + graph_hundred_as_thousand |= pynutil.add_weight( + pynutil.delete("ढाई") + + delete_space + + pynutil.insert("२५००", weight=-0.1) + + delete_space + + delete_thousand, + -0.1, + ) + + graph_in_hundreds = pynutil.add_weight( + pynutil.delete("साढ़े") + + delete_space + + (graph_digit | self.graph_two_digit) + + pynutil.insert("५०", weight=-0.1) + + delete_space + + delete_hundred, + -0.1, + ) + graph_in_hundreds |= pynutil.add_weight( + pynutil.delete("सवा") + + delete_space + + (graph_digit | self.graph_two_digit) + + pynutil.insert("२५", weight=-0.1) + + delete_space + + delete_hundred, + -0.1, + ) + graph_in_hundreds |= pynutil.add_weight( + pynutil.delete("पौने") + + delete_space + + graph_paune + + pynutil.insert("७५", weight=-0.1) + + delete_space + + delete_hundred, + -0.1, + ) + graph_in_hundreds |= pynutil.add_weight( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + pynutil.insert("१५०", weight=-0.1) + + delete_space + + delete_hundred, + -0.1, + ) + graph_in_hundreds |= pynutil.add_weight( + pynutil.delete("ढाई") + delete_space + pynutil.insert("२५०", weight=-0.1) + delete_space + delete_hundred, + -0.1, + ) + self.graph_hundreds = graph_hundred_component | graph_hundred_as_thousand | graph_in_hundreds + + graph_teens_and_ties_component = pynini.union( + graph_teens_and_ties | pynutil.insert("००") + delete_space + (graph_digit | pynutil.insert("०")), + ) + graph_ties_component_at_least_one_none_zero_digit = self.graph_two_digit @ ( + pynini.closure(NEMO_HI_DIGIT) + pynini.closure(NEMO_HI_DIGIT) + ) + self.graph_ties_component_at_least_one_none_zero_digit = graph_ties_component_at_least_one_none_zero_digit + + # %% Indian numeric format simple https://en.wikipedia.org/wiki/Indian_numbering_system + # This only covers "standard format". + # Conventional format like thousand crores/lakh crores is yet to be implemented + graph_in_thousands = pynini.union( + self.graph_two_digit + delete_space + delete_thousand, + pynutil.insert("००", weight=0.1), + ) + self.graph_thousands = graph_in_thousands + + graph_in_lakhs = pynini.union( + self.graph_two_digit + delete_space + pynutil.delete("लाख"), + pynutil.insert("००", weight=0.1), + ) + + graph_in_crores = pynini.union( + self.graph_two_digit + delete_space + pynutil.delete("करोड़") | pynutil.delete("crores"), + pynutil.insert("००", weight=0.1), + ) + + graph_in_arabs = pynini.union( + self.graph_two_digit + delete_space + pynutil.delete("अरब"), + pynutil.insert("००", weight=0.1), + ) + + graph_in_kharabs = pynini.union( + self.graph_two_digit + delete_space + pynutil.delete("खरब"), + pynutil.insert("००", weight=0.1), + ) + + graph_in_nils = pynini.union( + self.graph_two_digit + delete_space + pynutil.delete("नील"), + pynutil.insert("००", weight=0.1), + ) + + graph_in_padmas = pynini.union( + self.graph_two_digit + delete_space + pynutil.delete("पद्म"), + pynutil.insert("००", weight=0.1), + ) + + graph_in_shankhs = pynini.union( + self.graph_two_digit + delete_space + pynutil.delete("शंख"), + pynutil.insert("००", weight=0.1), + ) + + graph_ind = ( + graph_in_shankhs + + delete_space + + graph_in_padmas + + delete_space + + graph_in_nils + + delete_space + + graph_in_kharabs + + delete_space + + graph_in_arabs + + delete_space + + graph_in_crores + + delete_space + + graph_in_lakhs + + delete_space + + graph_in_thousands + ) + graph_no_prefix = pynutil.add_weight( + pynini.cross("सौ", "१००") + | pynini.cross("हज़ार", "१०००") + | pynini.cross("हजार", "१०००") + | pynini.cross("लाख", "१०००००") + | pynini.cross("करोड़", "१०००००००"), + 2, + ) + + graph = pynini.union( + graph_ind + delete_space + self.graph_hundreds, graph_zero, graph_no_prefix + ) # graph_digit_plus_hundred, + + graph = graph @ pynini.union( + pynutil.delete(pynini.closure("०")) + + pynini.difference(NEMO_HI_DIGIT, "०") + + pynini.closure(NEMO_HI_DIGIT), + "०", + ) + + labels_exception = [pynini.string_file(get_abs_path("data/numbers/labels_exception.tsv"))] + + graph_exception = pynini.union(*labels_exception).optimize() + + self.graph_no_exception = graph + + self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + + optional_minus_graph = pynini.closure( + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"-\"") + NEMO_SPACE, 0, 1 + ) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/date.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/date.py new file mode 100644 index 000000000..d019a7973 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/date.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + DEVANAGARI_DIGIT, + NEMO_HI_DIGIT, + GraphFst, + delete_extra_space, + delete_space, + insert_space, + integer_to_devanagari, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, + e.g. पांच जनवरी दो हज़ार बारह -> date { month: "जनवरी" day: "५" year: "२०१२" preserve_order: true } + e.g. दो हज़ार बारह -> date { year: "२०१२" preserve_order: true } + Args: + cardinal: CardinalFst + date: DateFst + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst): + super().__init__(name="date", kind="classify") + + graph_year = pynutil.add_weight( + pynini.compose(cardinal.graph_no_exception, pynini.closure(NEMO_HI_DIGIT, 1, 4)), 0.03 + ) + cardinal_graph = cardinal.graph_no_exception + + month_graph = pynini.string_file(get_abs_path("data/date/months.tsv")) + + graph_date_days = cardinal.graph_digit | cardinal.graph_teens_and_ties + date_days = pynini.union(*[integer_to_devanagari(i) for i in range(1, 32)]).optimize() + graph_date_days = graph_date_days @ date_days + + graph_century = pynini.string_file(get_abs_path("data/date/century.tsv")).invert() + graph_morph_features = pynini.string_file(get_abs_path("data/ordinals/morph_features.tsv")) + + self.day = pynutil.insert("day: \"") + graph_date_days + pynutil.insert("\" ") + self.month = pynutil.insert("month: \"") + month_graph + pynutil.insert("\" ") + self.year = pynutil.insert("year: \"") + graph_year + pynutil.insert("\" ") + self.year_range = ( + pynutil.insert("year: \"") + + graph_year + + delete_space + + pynini.cross("से", "-") + + delete_space + + graph_year + + delete_space + + pynutil.insert("\" ") + ) + self.century = pynutil.insert("text: \"") + graph_century + pynutil.insert("\" ") + self.ordinal_century = pynutil.insert("era: \"") + cardinal_graph + pynutil.insert("\" ") + self.morpho_graph = ( + pynutil.insert("morphosyntactic_features: \"") + graph_morph_features + pynutil.insert("\"") + ) + graph_day_month = self.day + delete_space + self.month + graph_month_day = self.month + delete_space + self.day + graph_month_day += pynutil.insert(" preserve_order: true") + graph_day_month_year = self.day + delete_space + self.month + delete_space + self.year + graph_month_day_year = self.month + delete_space + self.day + delete_space + self.year + graph_month_day_year += pynutil.insert(" preserve_order: true") + graph_month_year = self.month + delete_space + self.year + graph_saal = self.year + graph_AD_BC = self.year + delete_space + self.century + graph_day_month_year_century = ( + self.day + delete_space + self.month + delete_space + self.year + delete_space + self.century + ) + graph_month_year_century = self.month + delete_space + self.year + delete_space + self.century + graph_year_range = self.year_range + graph_year_range_century = self.year_range + delete_space + self.century + + graph_ordinal_century = self.ordinal_century + self.morpho_graph + delete_extra_space + self.century + + graph_ordinal_century = self.ordinal_century + self.morpho_graph + delete_extra_space + self.century + graph_date_exceptions = self.month + delete_space + pynutil.delete("की") + delete_space + self.day + graph_date_exceptions += pynutil.insert("preserve_order: true") + + graph = ( + graph_day_month + | graph_month_day + | graph_day_month_year + | graph_month_day_year + | graph_month_year + | graph_saal + | graph_AD_BC + | graph_day_month_year_century + | graph_month_year_century + | graph_year_range + | graph_year_range_century + | graph_ordinal_century + | graph_date_exceptions + ) + final_graph = self.add_tokens(graph) + self.fst = final_graph diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/decimal.py new file mode 100644 index 000000000..ddbf32c9b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/decimal.py @@ -0,0 +1,127 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.import pynini + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + MIN_NEG_WEIGHT, + MINUS, + NEMO_DIGIT, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + capitalized_input_graph, + delete_extra_space, + delete_space, +) +from nemo_text_processing.text_normalization.en.utils import load_labels + + +def get_quantity( + decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike', input_case: str = INPUT_LOWER_CASED +) -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. दस लाख -> integer_part: "१॰" quantity: "लाख" + e.g. एक दशमलव पाँच लाख -> integer_part: "१" fractional_part: "५" quantity: "लाख" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + input_case: accepting either "lower_cased" or "cased" input. + """ + numbers = cardinal_up_to_hundred @ ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + + suffix = pynini.string_file(get_abs_path("data/numbers/thousands.tsv")) + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + delete_extra_space + + pynutil.insert("quantity: \"") + + suffix + + pynutil.insert("\"") + ) + res |= decimal + delete_extra_space + pynutil.insert("quantity: \"") + suffix + pynutil.insert("\"") + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal + Decimal point "." is determined by "दशमलव" + e.g. ऋण एक दशमलव दो छह -> decimal { negative: "true" integer_part: "१" morphosyntactic_features: "." fractional_part: "२६" } + + + This decimal rule assumes that decimals can be pronounced as: + (a cardinal) + ('दशमलव') plus (any sequence of cardinals <१०००, including 'शून्य') + + Also writes large numbers in shortened form, e.g. + e.g. एक दशमलव दो छह लाख -> decimal { negative: "false" integer_part: "१" morphosyntactic_features: "." fractional_part: "२६" quantity: "लाख" } + e.g. दो लाख -> decimal { negative: "false" integer_part: "२" quantity: "लाख" } + e.g. एक अरब आठ सौ चौबीस लाख -> decimal { negative: "false" integer_part: "१८२४" quantity: "लाख" } + Args: + cardinal: CardinalFst + + """ + + def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="decimal", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + + graph_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + graph_decimal |= pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + + graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal + self.graph = graph_decimal + + point = pynutil.delete("दशमलव") + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("ऋण", "\"true\"") + delete_extra_space, + 0, + 1, + ) + + graph_fractional = pynutil.insert("fractional_part: \"") + graph_decimal + pynutil.insert("\"") + graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure(graph_integer + delete_extra_space, 0, 1) + point + delete_extra_space + graph_fractional + ) + final_graph = optional_graph_negative + final_graph_wo_sign + + self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( + final_graph_wo_sign, cardinal_graph, input_case=input_case + ) + + # accept semiotic spans that start with a capital letter + self.final_graph_wo_negative |= pynutil.add_weight( + pynini.compose(TO_LOWER + NEMO_SIGMA, self.final_graph_wo_negative).optimize(), MIN_NEG_WEIGHT + ) + + quantity_graph = get_quantity(final_graph_wo_sign, cardinal_graph, input_case=input_case) + final_graph |= optional_graph_negative + quantity_graph + + if input_case == INPUT_CASED: + final_graph = capitalized_input_graph(final_graph) + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/fraction.py new file mode 100644 index 000000000..970bf7313 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/fraction.py @@ -0,0 +1,164 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.utils import apply_fst, get_abs_path +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.text_normalization.hi.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + MIN_NEG_WEIGHT, + MINUS, + NEMO_HI_DIGIT, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + capitalized_input_graph, + delete_extra_space, + delete_space, +) + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + Fraction "/" is determined by "बटा" + e.g. ऋण एक बटा छब्बीस -> fraction { negative: "true" numerator: "१" denominator: "२६" } + e.g. छह सौ साठ बटा पाँच सौ तैंतालीस -> fraction { negative: "false" numerator: "६६०" denominator: "५४३" } + + + The fractional rule assumes that fractions can be pronounced as: + (a cardinal) + ('बटा') plus (a cardinal, excluding 'शून्य') + Args: + cardinal: CardinalFst + fraction: FractionFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="fraction", kind="classify") + # integer_part # numerator # denominator + graph_cardinal = cardinal.graph_no_exception + + integer = pynutil.insert("integer_part: \"") + graph_cardinal + pynutil.insert("\" ") + integer += delete_space + delete_bata = pynini.union(pynutil.delete(" बटा ") | pynutil.delete(" बटे ")) + + numerator = pynutil.insert("numerator: \"") + graph_cardinal + pynutil.insert("\"") + denominator = pynutil.insert(" denominator: \"") + graph_cardinal + pynutil.insert("\"") + + graph_fraction = numerator + delete_bata + denominator + graph_mixed_fraction = integer + delete_extra_space + pynutil.delete("सही") + delete_space + graph_fraction + + graph_saade = pynutil.add_weight( + pynutil.delete("साढ़े") + + delete_space + + integer + + pynutil.insert("numerator: \"१\"") + + delete_space + + pynutil.insert(" denominator: \"२\""), + -0.01, + ) + graph_sava = pynutil.add_weight( + pynutil.delete("सवा") + + delete_space + + integer + + pynutil.insert("numerator: \"१\"") + + delete_space + + pynutil.insert(" denominator: \"४\""), + -0.001, + ) + graph_paune = pynutil.add_weight( + pynutil.delete("पौने") + + delete_space + + integer + + pynutil.insert("numerator: \"३\"") + + delete_space + + pynutil.insert(" denominator: \"४\""), + -0.01, + ) + graph_dedh = pynutil.add_weight( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + pynutil.insert("integer_part: \"१\"") + + pynutil.insert(" numerator: \"१\"") + + delete_space + + pynutil.insert(" denominator: \"२\""), + -0.01, + ) + graph_dhaai = pynutil.add_weight( + pynutil.delete("ढाई") + + delete_space + + pynutil.insert("integer_part: \"२\"") + + pynutil.insert(" numerator: \"१\"") + + delete_space + + pynutil.insert(" denominator: \"२\""), + -0.1, + ) + + graph_aadha_and_saade_only = ( + pynini.union(pynutil.delete("आधा") | pynutil.delete("साढ़े")) + + delete_space + + pynutil.insert(" numerator: \"१\"") + + delete_space + + pynutil.insert(" denominator: \"२\"") + ) + graph_sava_only = ( + pynutil.delete("सवा") + + delete_space + + pynutil.insert(" numerator: \"१\"") + + delete_space + + pynutil.insert(" denominator: \"४\"") + ) + graph_paune_only = ( + pynini.union(pynutil.delete("पौन") | pynutil.delete("पौना")) + + delete_space + + pynutil.insert("numerator: \"३\"") + + delete_space + + pynutil.insert(" denominator: \"४\"") + ) + + graph_tihaai = ( + numerator + delete_space + pynutil.delete("तिहाई") + delete_space + pynutil.insert(" denominator: \"३\"") + ) + graph_chauthaai = ( + numerator + delete_space + pynutil.delete("चौथाई") + delete_space + pynutil.insert(" denominator: \"४\"") + ) + + graph_quarterly_exceptions = ( + graph_saade + | graph_sava + | graph_paune + | graph_dedh + | graph_dhaai + | graph_aadha_and_saade_only + | graph_sava_only + | graph_paune_only + | graph_tihaai + | graph_chauthaai + ) + + graph = graph_fraction | graph_mixed_fraction | graph_quarterly_exceptions + self.graph = graph.optimize() + self.final_graph_wo_negative = graph + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("ऋण", "\"true\"") + delete_extra_space, + 0, + 1, + ) + graph = optional_graph_negative + graph + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/measure.py new file mode 100644 index 000000000..59227a436 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/measure.py @@ -0,0 +1,254 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_CHAR, + NEMO_WHITE_SPACE, + GraphFst, + convert_space, + delete_extra_space, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. ऋण बारह किलोग्राम -> measure { decimal { negative: "true" integer_part: "१२" fractional_part: "५०"} units: "kg" } + e.g. ऋण बारह किलोग्राम -> measure { cardinal { negative: "true" integer_part: "१२"} units: "kg" } + e.g. सात शून्य शून्य ओक स्ट्रीट -> measure { units: "address" cardinal { integer: "७०० ओक स्ट्रीट" } preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="measure", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + decimal_graph = decimal.final_graph_wo_negative + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("ऋण", "\"true\"") + delete_extra_space, + 0, + 1, + ) + + measurements_graph = pynini.string_file(get_abs_path("data/measure/measurements.tsv")).invert() + paune_graph = pynini.string_file(get_abs_path("data/numbers/paune.tsv")).invert() + + self.measurements = pynutil.insert("units: \"") + measurements_graph + pynutil.insert("\" ") + graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + graph_integer_paune = pynutil.insert("integer_part: \"") + paune_graph + pynutil.insert("\"") + + graph_saade_single_digit = pynutil.add_weight( + pynutil.delete("साढ़े") + + delete_space + + graph_integer + + delete_space + + pynutil.insert(" fractional_part: \"५\""), + 0.1, + ) + graph_sava_single_digit = pynutil.add_weight( + pynutil.delete("सवा") + + delete_space + + graph_integer + + delete_space + + pynutil.insert(" fractional_part: \"२५\""), + 0.1, + ) + graph_paune_single_digit = pynutil.add_weight( + pynutil.delete("पौने") + + delete_space + + graph_integer_paune + + delete_space + + pynutil.insert(" fractional_part: \"७५\""), + 1, + ) + graph_dedh_single_digit = pynutil.add_weight( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + pynutil.insert("integer_part: \"१\"") + + delete_space + + pynutil.insert(" fractional_part: \"५\""), + 0.1, + ) + graph_dhaai_single_digit = pynutil.add_weight( + pynutil.delete("ढाई") + + delete_space + + pynutil.insert("integer_part: \"२\"") + + delete_space + + pynutil.insert(" fractional_part: \"५\""), + 1, + ) + + graph_exceptions = ( + graph_saade_single_digit + | graph_sava_single_digit + | graph_paune_single_digit + | graph_dedh_single_digit + | graph_dhaai_single_digit + ) + + graph_measurements = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal_graph + + pynutil.insert(" }") + + delete_extra_space + + self.measurements + ) + graph_measurements |= ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + self.measurements + ) + graph_quarterly_measurements = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + graph_exceptions + + pynutil.insert(" }") + + delete_extra_space + + self.measurements + ) + graph_exception_bai = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + delete_space + + pynini.cross("बाई", "x") + + delete_space + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + pynini.closure(delete_extra_space + self.measurements) + ) + + # Shared digit word -> Devanagari digit mapping + num_word = ( + ( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + | pynini.string_file(get_abs_path("data/telephone/eng_digit.tsv")) + | pynini.string_file(get_abs_path("data/telephone/eng_zero.tsv")) + ) + .invert() + .optimize() + ) + + delete_one_space = pynutil.delete(" ") + + # Structured address: state/city + pincode + states = pynini.string_file(get_abs_path("data/address/states.tsv")) + cities = pynini.string_file(get_abs_path("data/address/cities.tsv")) + state_city_names = pynini.union(states, cities).optimize() + + pincode = num_word + pynini.closure(delete_one_space + num_word, 5, 5) + + structured_pattern = ( + state_city_names + + pynini.closure(pynini.accep(",") + pynini.accep(" ") + state_city_names, 0, 1) + + pynini.accep(" ") + + pincode + ).optimize() + + structured_address_graph = ( + pynutil.insert('units: "address" cardinal { integer: "') + + convert_space(structured_pattern) + + pynutil.insert('" } preserve_order: true') + ) + structured_address_graph = pynutil.add_weight(structured_address_graph, 1.0).optimize() + + # Address: digit/special/ordinal conversion with context keywords + special_word = pynini.string_file(get_abs_path("data/address/special_characters.tsv")) + ordinal_word = pynini.string_file(get_abs_path("data/address/ordinals.tsv")) + context_keywords_fsa = pynini.string_file(get_abs_path("data/address/context_cues.tsv")) + + digit_passthrough = pynini.string_file(get_abs_path("data/address/digit_passthrough.tsv")).optimize() + digit_unit = pynini.union(num_word, digit_passthrough).optimize() + + all_digit_inputs = pynini.project(digit_unit, "input").optimize() + all_ordinal_inputs = pynini.project(ordinal_word, "input").optimize() + + non_space_non_comma = pynini.difference( + NEMO_CHAR, pynini.union(NEMO_WHITE_SPACE, pynini.accep(",")) + ).optimize() + any_word = pynini.closure(non_space_non_comma, 1).optimize() + + text_word = pynini.difference(any_word, pynini.union(all_digit_inputs, all_ordinal_inputs)).optimize() + + digit_block = digit_unit + pynini.closure(pynutil.add_weight(delete_one_space + digit_unit, -1.0)) + + connector = delete_one_space + special_word + delete_one_space + + matchable = pynini.union( + pynutil.add_weight(digit_block, -0.1), + pynutil.add_weight(ordinal_word, -0.2), + pynutil.add_weight(text_word, 0.1), + ).optimize() + + chain = matchable + pynini.closure(pynutil.add_weight(connector + matchable, -0.5)) + + opt_comma = pynini.closure(pynini.accep(","), 0, 1) + element = chain + opt_comma + address_content = element + pynini.closure(pynini.accep(" ") + element) + + # Context detection: keyword must appear as a complete word in the input + any_char = pynini.union( + pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE), + NEMO_WHITE_SPACE, + ).optimize() + sigma_star = pynini.closure(any_char).optimize() + + word_sep = pynini.union(pynini.accep(" "), pynini.accep(",")).optimize() + + input_pattern = pynini.union( + context_keywords_fsa + word_sep + sigma_star, + sigma_star + pynini.accep(" ") + context_keywords_fsa, + sigma_star + pynini.accep(" ") + context_keywords_fsa + word_sep + sigma_star, + context_keywords_fsa, + ).optimize() + + address_graph = pynini.compose(input_pattern, address_content).optimize() + + address_graph = ( + pynutil.insert('units: "address" cardinal { integer: "') + + convert_space(address_graph) + + pynutil.insert('" } preserve_order: true') + ) + address_graph = pynutil.add_weight(address_graph, 1.05).optimize() + + graph = ( + graph_measurements + | graph_quarterly_measurements + | graph_exception_bai + | address_graph + | structured_address_graph + ) + self.graph = graph.optimize() + + final_graph = self.add_tokens(graph) + self.fst = final_graph diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/money.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/money.py new file mode 100644 index 000000000..e79c9d0b3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/money.py @@ -0,0 +1,254 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + GraphFst, + convert_space, + delete_extra_space, + delete_space, + insert_space, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. बहत्तर लाइटकॉइन -> money { integer_part: "७२" currency: "ł" } + e.g. बहत्तर मोनेरो -> money { integer_part: "७२" currency: "ɱ" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + money: MoneyFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="money", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + cardinal_single_and_double_digit_graph = cardinal.graph_digit | cardinal.graph_teens_and_ties + decimal_graph = decimal.final_graph_wo_negative + currency_graph = pynini.string_file(get_abs_path("data/money/currency.tsv")).invert() + paune_graph = pynini.string_file(get_abs_path("data/numbers/paune.tsv")).invert() + + self.integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + self.integer_quarterly_measures = pynutil.insert("integer_part: \"") + cardinal_single_and_double_digit_graph + self.integer_paune = pynutil.insert("integer_part: \"") + paune_graph + self.paise = pynutil.insert("fractional_part: \"") + cardinal_graph + pynutil.insert("\"") + self.fraction = decimal_graph + self.currency = pynutil.insert("currency: \"") + currency_graph + pynutil.insert("\" ") + aur = pynutil.delete("और") + delete_hundred = pynutil.delete("सौ") + delete_lakh = pynutil.delete("लाख") + delete_hazar = pynutil.delete("हजार") | pynutil.delete("हज़ार") + delete_crore = pynutil.delete("करोड़") | pynutil.delete("करोड़") + + graph_currency_decimal = self.fraction + delete_extra_space + self.currency + graph_currency_cardinal = self.integer + delete_extra_space + self.currency + + graph_rupay_and_paisa = ( + graph_currency_cardinal + + delete_extra_space + + pynini.closure(aur + delete_extra_space, 0, 1) + + self.paise + + delete_extra_space + + pynutil.delete(currency_graph) + ) + # cases for saade,sava with teens and ties + graph_saade_teens_ties = ( + pynutil.delete("साढ़े") + + delete_space + + self.integer_quarterly_measures + + pynutil.insert("\"") + + delete_space + + pynutil.insert(" fractional_part: \"५०\"") + + delete_extra_space + + self.currency + ) + graph_sava_teens_ties = ( + pynutil.delete("सवा") + + delete_space + + self.integer_quarterly_measures + + pynutil.insert("\"") + + delete_space + + pynutil.insert(" fractional_part: \"२५\"") + + delete_extra_space + + self.currency + ) + graph_dedh = ( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + pynutil.insert("integer_part: \"१\"") + + delete_space + + pynutil.insert(" fractional_part: \"५०\"") + + delete_extra_space + + self.currency + ) + graph_dhaai = ( + pynutil.delete("ढाई") + + delete_space + + pynutil.insert("integer_part: \"२\"") + + delete_space + + pynutil.insert(" fractional_part: \"५०\"") + + delete_extra_space + + self.currency + ) + + graph_exceptions_teens_ties = graph_saade_teens_ties | graph_sava_teens_ties | graph_dedh | graph_dhaai + + # cases for saade,sava,paune,dedh and dhaai with hundreds and thousands + graph_exceptions = self.integer + delete_extra_space + self.currency + + # exceptions with lakhs + graph_saade_lakh = pynutil.add_weight( + pynutil.delete("साढ़े") + + delete_space + + self.integer_quarterly_measures + + delete_space + + pynutil.insert("५००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_lakh + + delete_extra_space + + self.currency, + 0.01, + ) + graph_sava_lakh = pynutil.add_weight( + pynutil.delete("सवा") + + delete_space + + self.integer_quarterly_measures + + delete_space + + pynutil.insert("२५०००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_lakh + + delete_extra_space + + self.currency, + 0.01, + ) + graph_paune_lakh = ( + pynutil.delete("पौने") + + delete_space + + self.integer_paune + + delete_space + + pynutil.insert("७५०००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_lakh + + delete_extra_space + + self.currency + ) + graph_dedh_lakh = ( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + pynutil.insert("integer_part: \"") + + pynutil.insert("१५००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_lakh + + delete_extra_space + + self.currency + ) + graph_dhaai_lakh = ( + pynutil.delete("ढाई") + + delete_space + + pynutil.insert("integer_part: \"") + + pynutil.insert("२५००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_lakh + + delete_extra_space + + self.currency + ) + + graph_exceptions_lakhs = ( + graph_saade_lakh | graph_sava_lakh | graph_paune_lakh | graph_dedh_lakh | graph_dhaai_lakh + ) + + # exceptions with crores + graph_saade_crore = ( + pynutil.delete("साढ़े") + + delete_space + + self.integer_quarterly_measures + + delete_space + + pynutil.insert("५००००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_crore + + delete_extra_space + + self.currency + ) + graph_sava_crore = ( + pynutil.delete("सवा") + + delete_space + + self.integer_quarterly_measures + + delete_space + + pynutil.insert("२५०००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_crore + + delete_extra_space + + self.currency + ) + graph_paune_crore = ( + pynutil.delete("पौने") + + delete_space + + self.integer_paune + + delete_space + + pynutil.insert("७५०००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_crore + + delete_extra_space + + self.currency + ) + graph_dhaai_crore = ( + pynutil.delete("ढाई") + + delete_space + + pynutil.insert("integer_part: \"") + + pynutil.insert("२५००००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_crore + + delete_extra_space + + self.currency + ) + graph_dedh_crore = ( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + pynutil.insert("integer_part: \"") + + pynutil.insert("१५००००००", weight=-0.1) + + pynutil.insert("\"") + + delete_space + + delete_crore + + delete_extra_space + + self.currency + ) + + graph_exceptions_crores = ( + graph_saade_crore | graph_sava_crore | graph_paune_crore | graph_dedh_crore | graph_dhaai_crore + ) + + graph_quarterly_measures = ( + graph_exceptions_teens_ties | graph_exceptions | graph_exceptions_lakhs | graph_exceptions_crores + ) + + graph = graph_currency_decimal | graph_currency_cardinal | graph_rupay_and_paisa | graph_quarterly_measures + self.graph = graph.optimize() + + final_graph = self.add_tokens(graph) + self.fst = final_graph diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/ordinal.py new file mode 100644 index 000000000..473055891 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/ordinal.py @@ -0,0 +1,88 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_CHAR, + NEMO_HI_DIGIT, + NEMO_SIGMA, + GraphFst, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + e.g. तेरहवां -> ordinal { integer: "१३" } + + Args: + cardinal: CardinalFst + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="ordinal", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")) + graph_teens = pynini.string_file(get_abs_path("data/ordinals/teens_and_ties.tsv")) + graph_digit_hundred = pynini.string_file(get_abs_path("data/ordinals/hundred_digit.tsv")) + graph = pynini.closure(NEMO_CHAR) + pynini.union( + graph_digit, + graph_teens, + graph_digit_hundred, + pynini.cross("वाँ", "वाँ"), + pynini.cross("वां", "वां"), + pynini.cross("वें", "वें"), + pynini.cross("वे", "वे"), + ) + + graph_fem_digit = pynini.string_file(get_abs_path("data/ordinals/digit_fem.tsv")) + graph_fem_teens = pynini.string_file(get_abs_path("data/ordinals/teens_and_ties_fem.tsv")) + graph_digit_hundred_fem = pynini.string_file(get_abs_path("data/ordinals/hundred_digit_fem.tsv")) + graph_fem = pynini.closure(NEMO_CHAR) + pynini.union( + graph_fem_digit, + graph_fem_teens, + graph_digit_hundred_fem, + pynini.cross("वीं", "वीं"), + pynini.cross("वी", "वी"), + ) + graph = pynini.compose( + graph | graph_fem, + ( + cardinal_graph + + pynini.union( + pynini.cross("वाँ", "वाँ"), + pynini.cross("वां", "वां"), + pynini.cross("वीं", "वीं"), + pynini.cross("वी", "वी"), + pynini.cross("वें", "वें"), + pynini.cross("वे", "वे"), + ) + ), + ).optimize() + + morph_features_graph = pynini.string_file(get_abs_path("data/ordinals/morph_features.tsv")) + morpho_graph = pynutil.insert("\" morphosyntactic_features: \"") + morph_features_graph + pynutil.insert("\"") + + rule = pynini.cdrewrite(morpho_graph, pynini.closure(NEMO_HI_DIGIT), pynini.union("[EOS]", " "), NEMO_SIGMA) + + final_graph = pynutil.insert("integer: \"") + graph @ rule + self.final_graph = self.add_tokens(final_graph) + final_graph = self.final_graph + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/punctuation.py new file mode 100644 index 000000000..96cefc506 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/punctuation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. आ, -> tokens { name: "आ" } tokens { name: "," } + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py new file mode 100644 index 000000000..ad584b58b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py @@ -0,0 +1,160 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_CHAR, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path + +shunya = ( + pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + | pynini.string_file(get_abs_path("data/telephone/eng_zero.tsv")).invert() +) +digit_without_shunya = ( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + | pynini.string_file(get_abs_path("data/telephone/eng_digit.tsv")).invert() +) +digit = digit_without_shunya | shunya + + +def get_context(keywords: list): + keywords = pynini.union(*keywords) + + # Load Hindi digits from TSV files + hindi_digits = ( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + ).project("output") + + # Load English digits from TSV files + english_digits = ( + pynini.string_file(get_abs_path("data/telephone/eng_digit.tsv")) + | pynini.string_file(get_abs_path("data/telephone/eng_zero.tsv")) + ).project("output") + + all_digits = hindi_digits | english_digits + + non_digit_char = pynini.difference(NEMO_CHAR, pynini.union(all_digits, NEMO_WHITE_SPACE)) + word = pynini.closure(non_digit_char, 1) + NEMO_WHITE_SPACE + window = pynini.closure(word, 0, 5) + before = (keywords + window).optimize() + after = (window + keywords).optimize() + + return before, after + + +def generate_context_graph(context_keywords, length): + context_before, context_after = get_context(context_keywords) + digits = pynini.closure(digit + delete_space, length - 1, length - 1) + digit + + graph_after_context = digits + NEMO_WHITE_SPACE + context_after + graph_before_context = context_before + NEMO_WHITE_SPACE + digits + graph_without_context = digits + + return ( + pynutil.insert("number_part: \"") + + (graph_before_context | graph_after_context | graph_without_context) + + pynutil.insert("\" ") + ).optimize() + + +def generate_pincode(context_keywords): + return generate_context_graph(context_keywords, 6) + + +def generate_credit(context_keywords): + return generate_context_graph(context_keywords, 4) + + +def generate_mobile(context_keywords): + context_before, context_after = get_context(context_keywords) + + country_code = pynini.cross("प्लस", "+") + pynini.closure(delete_space + digit, 2, 2) + NEMO_WHITE_SPACE + graph_country_code = ( + pynutil.insert("country_code: \"") + + (context_before + NEMO_WHITE_SPACE) ** (0, 1) + + country_code + + pynutil.insert("\" ") + ) + + number_part = digit_without_shunya + delete_space + pynini.closure(digit + delete_space, 8, 8) + digit + graph_number = ( + pynutil.insert("number_part: \"") + + number_part + + pynini.closure(NEMO_WHITE_SPACE + context_after, 0, 1) + + pynutil.insert("\" ") + ) + + graph = (graph_country_code + graph_number) | graph_number + return graph.optimize() + + +def generate_telephone(context_keywords): + context_before, context_after = get_context(context_keywords) + + landline = shunya + delete_space + pynini.closure(digit + delete_space, 9, 9) + digit + landline_with_context_before = context_before + NEMO_WHITE_SPACE + landline + landline_with_context_after = landline + NEMO_WHITE_SPACE + context_after + + return ( + pynutil.insert("number_part: \"") + + (landline | landline_with_context_before | landline_with_context_after) + + pynutil.insert("\" ") + ) + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying telephone numbers, e.g. + e.g. प्लस इक्यानवे नौ आठ सात छह पांच चार तीन दो एक शून्य => tokens { name: "+९१ ९८७६५ ४३२१०" } + Args: + Cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="telephone", kind="classify") + + # Load context cues from TSV file + context_cues = pynini.string_file(get_abs_path("data/telephone/context_cues.tsv")) + + # Extract keywords for each category + mobile_keywords = pynini.compose(pynutil.delete("mobile"), context_cues).project("output").optimize() + + landline_keywords = pynini.compose(pynutil.delete("landline"), context_cues).project("output").optimize() + + pincode_keywords = pynini.compose(pynutil.delete("pincode"), context_cues).project("output").optimize() + + credit_keywords = pynini.compose(pynutil.delete("credit"), context_cues).project("output").optimize() + + # Convert FSTs to keyword lists for generate_* functions + mobile = generate_mobile([mobile_keywords]) + landline = generate_telephone([landline_keywords]) + pincode = generate_pincode([pincode_keywords]) + credit = generate_credit([credit_keywords]) + + graph = ( + pynutil.add_weight(mobile, 0.7) + | pynutil.add_weight(landline, 0.8) + | pynutil.add_weight(credit, 0.9) + | pynutil.add_weight(pincode, 1) + ) + + self.final = graph.optimize() + self.fst = self.add_tokens(self.final) diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/time.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/time.py new file mode 100644 index 000000000..942b5022b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/time.py @@ -0,0 +1,158 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + DEVANAGARI_DIGIT, + GraphFst, + delete_extra_space, + delete_space, + insert_space, + integer_to_devanagari, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, + e.g. एक बजके सात मिनट -> time { hours: "१" minutes: "७" } + e.g. चार बजे चवालीस मिनट -> time { hours: "४" minutes: "४४" } + Args: + cardinal: CardinalFst + time: TimeFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="time", kind="classify") + + hour_graph = cardinal.graph_digit | cardinal.graph_teens_and_ties + time_hours = pynini.union(*[integer_to_devanagari(i) for i in range(1, 25)]).optimize() + hour_graph = hour_graph @ time_hours + + cardinal_graph = cardinal.graph_single_digit_with_zero | cardinal.graph_teens_and_ties + paune_hour_graph = pynini.string_file(get_abs_path("data/time/hour_for_paune.tsv")).invert() + + delete_baje = pynini.union( + pynutil.delete("बजके") | pynutil.delete("बजकर") | pynutil.delete("बजे") | pynutil.delete("घंटा") + ) + + delete_minute = pynutil.delete("मिनट") + delete_second = pynutil.delete("सेकंड") + + self.hour = pynutil.insert("hours: \"") + hour_graph + pynutil.insert("\" ") + self.paune_hour = pynutil.insert("hours: \"") + paune_hour_graph + pynutil.insert("\" ") + self.minute = pynutil.insert("minutes: \"") + cardinal_graph + pynutil.insert("\" ") + self.second = pynutil.insert("seconds: \"") + cardinal_graph + pynutil.insert("\" ") + + # hour minute second + graph_hms = ( + self.hour + + delete_space + + delete_baje + + delete_space + + self.minute + + delete_space + + delete_minute + + delete_space + + self.second + + delete_space + + delete_second + ) + + # hour minute and hour minute without "baje and minat" + graph_hm = pynutil.add_weight( + self.hour + + delete_space + + pynini.closure(delete_baje, 0, 1) + + delete_space + + self.minute + + pynini.closure(delete_space + delete_minute, 0, 1), + 0.01, + ) + + # hour second + graph_hs = pynutil.add_weight( + self.hour + delete_space + delete_baje + delete_space + self.second + delete_space + delete_second, 0.01 + ) + + # minute second + graph_ms = ( + self.minute + delete_space + delete_minute + delete_space + self.second + delete_space + delete_second + ) + + # hour + graph_hour = self.hour + delete_space + delete_baje + + graph_saade = pynutil.add_weight( + pynutil.delete("साढ़े") + + delete_space + + self.hour + + delete_space + + pynutil.insert(" minutes: \"३०\"") + + delete_space + + pynini.closure(delete_baje), + 0.01, + ) + graph_sava = pynutil.add_weight( + pynutil.delete("सवा") + + delete_space + + self.hour + + delete_space + + pynutil.insert(" minutes: \"१५\"") + + delete_space + + pynini.closure(delete_baje), + 0.01, + ) + graph_paune = pynutil.add_weight( + pynutil.delete("पौने") + + delete_space + + self.paune_hour + + delete_space + + pynutil.insert(" minutes: \"४५\"") + + delete_space + + pynini.closure(delete_baje), + 0.01, + ) + graph_dedh = pynutil.add_weight( + pynini.union(pynutil.delete("डेढ़") | pynutil.delete("डेढ़")) + + delete_space + + delete_baje + + pynutil.insert("hours: \"१\"") + + delete_space + + pynutil.insert(" minutes: \"३०\""), + 0.01, + ) + graph_dhaai = pynutil.add_weight( + pynutil.delete("ढाई") + + delete_space + + delete_baje + + pynutil.insert("hours: \"२\"") + + delete_space + + pynutil.insert(" minutes: \"३०\""), + 0.01, + ) + graph_quarterly_measures = ( + graph_dedh + | graph_dhaai + | ((graph_saade | graph_sava | graph_paune) + pynini.closure(delete_space + delete_baje)) + ) + + graph = graph_hms | graph_hm | graph_hs | graph_ms | graph_hour | graph_quarterly_measures + self.graph = graph.optimize() + + final_graph = self.add_tokens(graph) + self.fst = final_graph diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..50abab0e5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/tokenize_and_classify.py @@ -0,0 +1,123 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.inverse_text_normalization.hi.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + input_case: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"hi_itn.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + ordinal_graph = ordinal.fst + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + fraction = FractionFst(cardinal) + fraction_graph = fraction.fst + date = DateFst(cardinal, ordinal) + date_graph = date.fst + time = TimeFst(cardinal) + time_graph = time.fst + measure = MeasureFst(cardinal, decimal) + measure_graph = measure.fst + money = MoneyFst(cardinal, decimal) + money_graph = money.fst + telephone = TelephoneFst(cardinal) + telephone_graph = telephone.fst + punct_graph = PunctuationFst().fst + whitelist_graph = WhiteListFst().fst + word_graph = WordFst().fst + + classify = ( + pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.1) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + | pynutil.add_weight(whitelist_graph, 1.01) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/whitelist.py new file mode 100644 index 000000000..caeab03b1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/whitelist.py @@ -0,0 +1,57 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + GraphFst, + convert_space, + string_map_cased, +) +from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path +from nemo_text_processing.text_normalization.en.utils import load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + e.g. श्रीमती -> tokens { name: "श्रीमती" } + This class has highest priority among all classifier grammars. + Whitelisted tokens are defined and loaded from "data/whitelist.tsv" (unless input_file specified). + + Args: + input_file: path to a file with whitelist replacements (each line of the file: written_form\tspoken_form\n), + e.g. nemo_text_processing/inverse_text_normalization/en/data/whitelist.tsv + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__(self, input_case: str = INPUT_LOWER_CASED, input_file: str = None): + super().__init__(name="whitelist", kind="classify") + + if input_file is None: + input_file = get_abs_path("data/whitelist/whitelist.tsv") + + if not os.path.exists(input_file): + raise ValueError(f"Whitelist file {input_file} not found") + + whitelist = string_map_cased(input_file, input_case) + graph = pynutil.insert("name: \"") + convert_space(whitelist) + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/word.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/word.py new file mode 100644 index 000000000..8369ecdef --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. नींद -> tokens { name: "नींद" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/utils.py b/nemo_text_processing/inverse_text_normalization/hi/utils.py new file mode 100644 index 000000000..8e3f62c3c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/utils.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import logging +import os +import pynini + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path + + if not os.path.exists(abs_path): + logging.warning(f'{abs_path} does not exist') + return abs_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path, encoding="utf-8") + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +from pynini.lib import pynutil + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/cardinal.py new file mode 100644 index 000000000..c5d1d177a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/cardinal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. ऋण तेईस -> cardinal { integer: "२३" negative: "-" } -> -२३ + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space, + 0, + 1, + ) + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.numbers = graph + graph = optional_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/date.py new file mode 100644 index 000000000..7a5c10c4c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/date.py @@ -0,0 +1,152 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_space, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { month: "जनवरी" day: "५" year: "२०१२" preserve_order: true } -> जनवरी ५ २०१२ + date { day: "५" month: "जनवरी" year: "२०१२" preserve_order: true } -> ५ जनवरी २०१२ + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst): + super().__init__(name="date", kind="verbalize") + month = ( + pynutil.delete("month:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + day = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + delete_space + + pynutil.delete("\"") + ) + period = ( + pynutil.delete("text:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + era = ( + pynutil.delete("era:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + morpho_features = ( + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph_fy = year + graph_fy |= period + delete_space + year + + # century + graph_century = year + delete_extra_space + period + + # month (day) year + graph_mdy = month + delete_extra_space + day + pynutil.insert(",") + delete_extra_space + year + + # (day) month year + graph_dmy = day + delete_extra_space + month + pynutil.insert(",") + delete_extra_space + year + + # day month year century + graph_dmyc = ( + day + + delete_extra_space + + month + + pynutil.insert(",") + + delete_extra_space + + year + + delete_extra_space + + period + ) + + # month year + graph_my = month + pynini.closure(delete_extra_space + year, 0, 1) + + # month year century + graph_myc = month + pynutil.insert(",") + delete_extra_space + year + delete_extra_space + period + + # month day + graph_md = month + pynini.closure(delete_extra_space + day, 0, 1) + + # day month + graph_dm = day + pynini.closure(delete_extra_space + month, 0, 1) + + # year range + graph_year_range = year + + # ordinal century + graph_ordinal_century = era + delete_space + morpho_features + delete_extra_space + period + + # graph_ordinal_range = graph_ordinal + delete_extra_space + period + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = ( + ( + graph_fy + | graph_mdy + | graph_dmy + | graph_my + | graph_md + | graph_dm + | graph_century + | graph_dmyc + | graph_myc + | graph_year_range + | graph_ordinal_century + ) + + delete_space + + optional_preserve_order + ) + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/decimal.py new file mode 100644 index 000000000..bd265551d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { negative: "true" integer_part: "१२" fractional_part: "५००६" quantity: "अरब" } -> -१२.५००६ अरब + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-") + delete_space, 0, 1) + integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_integer = pynini.closure(integer + delete_space, 0, 1) + fractional = ( + pynutil.insert(".") + + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_fractional = pynini.closure(fractional + delete_space, 0, 1) + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1) + graph = optional_integer + optional_fractional + optional_quantity + self.numbers = graph + graph = optional_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/fraction.py new file mode 100644 index 000000000..45b5832b5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/fraction.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.utils import apply_fst +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fractions + e.g. fraction { numerator: "८" denominator: "३" } -> "८/३" + """ + + def __init__(self): + super().__init__(name="fraction", kind="verbalize") + + optional_negative = pynutil.delete("negative: \"") + pynini.cross("True", "-") + pynutil.delete("\"") + optional_negative = pynini.closure(optional_negative + delete_space, 0, 1) + + integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + optional_integer_part = pynini.closure(integer_part + NEMO_SPACE, 0, 1) + + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + denominator = pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph = ( + optional_negative + optional_integer_part + numerator + delete_space + pynutil.insert("/") + denominator + ) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/measure.py new file mode 100644 index 000000000..dc8592ebf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/measure.py @@ -0,0 +1,85 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { negative: "true" cardinal { integer: "१२" } units: "kg" } -> -१२ kg + measure { decimal { negative: "true" integer_part: "१२" fractional_part: "५०"} units: "kg" } -> -१२.५० kg + measure { units: "address" cardinal { integer: "७०० ओक स्ट्रीट" } preserve_order: true } -> ७०० ओक स्ट्रीट + + Args: + decimal: DecimalFst + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="measure", kind="verbalize") + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-"), 0, 1) + + # Exclude "address" from regular unit matching so address path handles it + unit = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete("\"") + + pynini.difference(pynini.closure(NEMO_CHAR - " ", 1), pynini.accep("address")) + + pynutil.delete("\"") + + delete_space + ) + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_decimal = ( + pynutil.delete("decimal {") + delete_space + decimal.numbers + delete_space + pynutil.delete("}") + ) + graph_exception_bai = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert(" ") + unit + graph |= graph_exception_bai + pynini.closure(delete_space + pynutil.insert(" ") + unit) + + # Address verbalizer: units: "address" cardinal { integer: "..." } preserve_order: true + preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + address_graph = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete("\"address\"") + + delete_space + + graph_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + graph |= address_graph + + delete_tokens = self.delete_tokens(graph) + self.decimal = graph_decimal + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/money.py new file mode 100644 index 000000000..56624ffc7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/money.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space, insert_space + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + बहत्तर लाइटकॉइन -> money { integer_part: "७२" currency: "ł" } -> ł७२ + + Args: + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="money", kind="verbalize") + unit = ( + pynutil.delete("currency:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = ( + unit + + delete_space + + decimal.numbers + + delete_space + + pynini.closure(insert_space + unit + delete_space + decimal.numbers + delete_space, 0, 1) + ) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/ordinal.py new file mode 100644 index 000000000..94f280798 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/ordinal.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "१३" } -> १३वां + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("morphosyntactic_features:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + self.numbers = graph + graph = graph.optimize() + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/telephone.py new file mode 100644 index 000000000..fb2e5d618 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/telephone.py @@ -0,0 +1,54 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2025 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone, e.g. + telephone { number_part: "123-123-5678" } + -> 123-123-5678 + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="telephone", kind="verbalize") + + number_part = pynutil.delete("number_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + optional_country_code = pynini.closure( + pynutil.delete("country_code: \"") + + delete_space + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + pynini.accep(" "), + 0, + 1, + ) + optional_city_code = pynini.closure( + pynutil.delete("extension: \"") + + pynutil.insert("०") + + delete_space + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + pynini.accep(" "), + 0, + 1, + ) + delete_tokens = self.delete_tokens(optional_country_code + number_part) + delete_tokens |= self.delete_tokens(optional_city_code + number_part) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/time.py new file mode 100644 index 000000000..99820a781 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/time.py @@ -0,0 +1,108 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_CHAR, + NEMO_HI_DIGIT, + GraphFst, + delete_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + e.g. एक बजके सात मिनट -> time { hours: "१" minutes: "७" } + e.g. चार बजे चवालीस मिनट -> time { hours: "४" minutes: "४४" } + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_HI_DIGIT, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_HI_DIGIT, 1) + + pynutil.delete("\"") + ) + second = ( + pynutil.delete("seconds:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_HI_DIGIT, 1) + + pynutil.delete("\"") + ) + + graph_hour = hour + delete_space + pynutil.insert(":") + delete_space + pynutil.insert("००") + delete_space + + # hour minute second + graph_hms = ( + hour + + delete_space + + pynutil.insert(":") + + delete_space + + minute + + delete_space + + pynutil.insert(":") + + delete_space + + second + + delete_space + ) + + # hour minute + graph_hm = hour + delete_space + pynutil.insert(":") + delete_space + minute + delete_space + + # hour second + graph_hs = ( + hour + + delete_space + + pynutil.insert(":") + + delete_space + + pynutil.insert("००") + + delete_space + + pynutil.insert(":") + + second + + delete_space + ) + + # minute second + graph_ms = ( + pynutil.insert("००") + + delete_space + + pynutil.insert(":") + + delete_space + + minute + + delete_space + + pynutil.insert(":") + + delete_space + + second + + delete_space + ) + + graph = graph_hour | graph_hms | graph_hm | graph_hs | graph_ms + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py new file mode 100644 index 000000000..f1a6c55a3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize.py @@ -0,0 +1,67 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + ordinal = OrdinalFst() + ordinal_graph = ordinal.fst + decimal = DecimalFst() + decimal_graph = decimal.fst + fraction_graph = FractionFst().fst + date_graph = DateFst(cardinal, ordinal).fst + time_graph = TimeFst().fst + measure_graph = MeasureFst(cardinal, decimal).fst + money_graph = MoneyFst(cardinal, decimal).fst + telephone_graph = TelephoneFst(cardinal).fst + word_graph = WordFst().fst + whitelist_graph = WhiteListFst().fst + + graph = ( + cardinal_graph + | whitelist_graph + | word_graph + | ordinal_graph + | decimal_graph + | fraction_graph + | date_graph + | time_graph + | measure_graph + | money_graph + | telephone_graph + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize_final.py new file mode 100644 index 000000000..17dfebf64 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/verbalize_final.py @@ -0,0 +1,45 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "अब" } tokens { time { hours: "१२" minutes: "३०" } } tokens { name: "बज" } tokens { name: "गए" } tokens { name: "हैं" } -> अब १२:३० बज गए हैं + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/whitelist.py new file mode 100644 index 000000000..9f342eacd --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/whitelist.py @@ -0,0 +1,44 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + GraphFst, + delete_space, +) + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "श्रीमती" } -> श्रीमती + """ + + def __init__(self): + super().__init__(name="whitelist", kind="verbalize") + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/word.py new file mode 100644 index 000000000..a411b2ba8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/verbalizers/word.py @@ -0,0 +1,39 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2024 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + GraphFst, + delete_space, +) + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens + e.g. tokens { name: "नींद" } -> नींद + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/__init__.py b/nemo_text_processing/inverse_text_normalization/hi_en/__init__.py new file mode 100644 index 000000000..cfe932251 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.hi_en.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/graph_utils.py b/nemo_text_processing/inverse_text_normalization/hi_en/graph_utils.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/graph_utils.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/hi_en/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/hi_en/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..09d7f2b15 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/taggers/tokenize_and_classify.py @@ -0,0 +1,173 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.en.taggers.cardinal import CardinalFst as EnCardinalFst +from nemo_text_processing.inverse_text_normalization.en.taggers.date import DateFst as EnDateFst +from nemo_text_processing.inverse_text_normalization.en.taggers.decimal import DecimalFst as EnDecimalFst +from nemo_text_processing.inverse_text_normalization.en.taggers.electronic import ElectronicFst as EnElectronicFst +from nemo_text_processing.inverse_text_normalization.en.taggers.measure import MeasureFst as EnMeasureFst +from nemo_text_processing.inverse_text_normalization.en.taggers.money import MoneyFst as EnMoneyFst +from nemo_text_processing.inverse_text_normalization.en.taggers.ordinal import OrdinalFst as EnOrdinalFst +from nemo_text_processing.inverse_text_normalization.en.taggers.punctuation import PunctuationFst as EnPunctuationFst +from nemo_text_processing.inverse_text_normalization.en.taggers.telephone import TelephoneFst as EnTelephoneFst +from nemo_text_processing.inverse_text_normalization.en.taggers.time import TimeFst as EnTimeFst +from nemo_text_processing.inverse_text_normalization.en.taggers.whitelist import WhiteListFst as EnWhiteListFst +from nemo_text_processing.inverse_text_normalization.en.taggers.word import WordFst as EnWordFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.hi.taggers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with Hindi whitelist replacements. If None, defaults to the Hindi whitelist at + nemo_text_processing/inverse_text_normalization/hi/data/whitelist/whitelist.tsv + en_whitelist: path to a file with English whitelist replacements. If None, defaults to the English whitelist at + nemo_text_processing/inverse_text_normalization/en/data/whitelist.tsv + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + en_whitelist: str = None, + input_case: str = INPUT_LOWER_CASED, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"hi_en_itn_{input_case}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + fraction = FractionFst(cardinal) + fraction_graph = fraction.fst + + measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst + date_graph = DateFst(cardinal, ordinal).fst + word_graph = WordFst().fst + time_graph = TimeFst(cardinal).fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst + whitelist_graph = WhiteListFst(input_file=whitelist).fst + punct_graph = PunctuationFst().fst + telephone_graph = TelephoneFst(cardinal).fst + + en_cardinal = EnCardinalFst(input_case=input_case) + en_cardinal_graph = en_cardinal.fst + + en_ordinal = EnOrdinalFst(cardinal=en_cardinal, input_case=input_case) + en_ordinal_graph = en_ordinal.fst + + en_decimal = EnDecimalFst(cardinal=en_cardinal, input_case=input_case) + en_decimal_graph = en_decimal.fst + + en_measure_graph = EnMeasureFst(cardinal=en_cardinal, decimal=en_decimal, input_case=input_case).fst + en_date_graph = EnDateFst(ordinal=en_ordinal, input_case=input_case).fst + en_word_graph = EnWordFst().fst + en_time_graph = EnTimeFst(input_case=input_case).fst + en_money_graph = EnMoneyFst(cardinal=en_cardinal, decimal=en_decimal, input_case=input_case).fst + en_whitelist_graph = EnWhiteListFst(input_file=en_whitelist, input_case=input_case).fst + en_punct_graph = EnPunctuationFst().fst + en_electronic_graph = EnElectronicFst(input_case=input_case).fst + en_telephone_graph = EnTelephoneFst(cardinal=en_cardinal, input_case=input_case).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(en_whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(en_time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(en_date_graph, 1.09) + | pynutil.add_weight(decimal_graph, 1.09) + | pynutil.add_weight(en_decimal_graph, 1.09) + | pynutil.add_weight(fraction_graph, 1.09) + | pynutil.add_weight(measure_graph, 1.6) + | pynutil.add_weight(en_measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.6) + | pynutil.add_weight(en_cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.6) + | pynutil.add_weight(en_ordinal_graph, 1.09) + | pynutil.add_weight(money_graph, 1.6) + | pynutil.add_weight(en_money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.6) + | pynutil.add_weight(en_telephone_graph, 1.1) + | pynutil.add_weight(en_electronic_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + | pynutil.add_weight(en_word_graph, 120) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + en_punct = ( + pynutil.insert("tokens { ") + pynutil.add_weight(en_punct_graph, weight=1.3) + pynutil.insert(" }") + ) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + + token + + pynini.closure(pynutil.insert(" ") + punct | en_punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/utils.py b/nemo_text_processing/inverse_text_normalization/hi_en/utils.py new file mode 100644 index 000000000..2bcba780d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize.py new file mode 100644 index 000000000..81cc937a1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize.py @@ -0,0 +1,102 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.en.verbalizers.cardinal import CardinalFst as EnCardinalFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.date import DateFst as EnDateFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.decimal import DecimalFst as EnDecimalFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.electronic import ElectronicFst as EnElectronicFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.measure import MeasureFst as EnMeasureFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.money import MoneyFst as EnMoneyFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.ordinal import OrdinalFst as EnOrdinalFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.telephone import TelephoneFst as EnTelephoneFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.time import TimeFst as EnTimeFst +from nemo_text_processing.inverse_text_normalization.en.verbalizers.whitelist import WhiteListFst as EnWhiteListFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst() + ordinal_graph = ordinal.fst + + decimal = DecimalFst() + decimal_graph = decimal.fst + + fraction_graph = FractionFst().fst + + date_graph = DateFst(cardinal, ordinal).fst + time_graph = TimeFst().fst + measure_graph = MeasureFst(cardinal, decimal).fst + money_graph = MoneyFst(cardinal, decimal).fst + telephone_graph = TelephoneFst(cardinal).fst + whitelist_graph = WhiteListFst().fst + + en_cardinal = EnCardinalFst() + en_cardinal_graph = en_cardinal.fst + en_ordinal_graph = EnOrdinalFst().fst + en_decimal = EnDecimalFst() + en_decimal_graph = en_decimal.fst + en_measure_graph = EnMeasureFst(decimal=en_decimal, cardinal=en_cardinal).fst + en_money_graph = EnMoneyFst(decimal=en_decimal).fst + en_date_graph = EnDateFst().fst + en_whitelist_graph = EnWhiteListFst().fst + en_telephone_graph = EnTelephoneFst().fst + en_time_graph = EnTimeFst().fst + en_electronic_graph = EnElectronicFst().fst + + graph = ( + en_time_graph + | pynutil.add_weight(time_graph, 1.1) + | date_graph + | pynutil.add_weight(en_date_graph, 1.1) + | money_graph + | pynutil.add_weight(en_money_graph, 1.1) + | fraction_graph + | measure_graph + | pynutil.add_weight(en_measure_graph, 1.1) + | ordinal_graph + | pynutil.add_weight(en_ordinal_graph, 1.1) + | decimal_graph + | pynutil.add_weight(en_decimal_graph, 1.1) + | cardinal_graph + | pynutil.add_weight(en_cardinal_graph, 1.1) + | whitelist_graph + | pynutil.add_weight(en_whitelist_graph, 1.1) + | telephone_graph + | pynutil.add_weight(en_telephone_graph, 1.1) + | en_electronic_graph + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize_final.py new file mode 100644 index 000000000..05386f09d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi_en/verbalizers/verbalize_final.py @@ -0,0 +1,44 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hi.verbalizers.word import WordFst +from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/hy/__init__.py b/nemo_text_processing/inverse_text_normalization/hy/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/__init__.py b/nemo_text_processing/inverse_text_normalization/hy/data/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/currency.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/currency.tsv new file mode 100644 index 000000000..6caf930d0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/currency.tsv @@ -0,0 +1,53 @@ +aed դիրհամ +ARS արգենտինական պեսո +֏ դրամ +֏ հայկական դրամ +a$ ավստրալական դոլար +br բելառուսական ռուբլի +₿ բիթքոին +£ ֆունտ +£ բրիտանական ֆունտ +ca$ կանադական դոլար +元 չինական յեն +元 յեն +kr դանիական կրոն +$ դոլար +€ եվրո +₾ վրացական լարի +₾ լարի +₹ հնդկական ռուփի +₹ ռուփի +﷼ պարսկական ռիալ +﷼ ռիալ +₪ իսրայելական շեկել +₪ շեկել +¥ ճապոնական յեն +¥ յեն +₸ ղազախական տենգե +₸ տենգե +som ղրղզական սոմ +som սոմ +ل.ل լիբանանյան ֆունտ +ل.ل լիբանանյան լիրա +$ մեքսիկական պեսո +nz$ նորզելանդական դոլլր +kr նորվեգական կրոն +zł լեհական զլոտի +zł զլոտի +£ ֆունտ ստերլինգ +£ ֆունտ +₽ ռուսական ռուբլի +₽ ռուբլի +rsd սերբական դինար +s$ սինգապուրի դոլար +₩ կորեական վոն +kr շվեդական կրոն +chf շվեյցարական ֆրանկ +£s սիրիական ֆունտ +₺ թուրքական լիրա +₴ ուկրաինական գրիվնա +$ ամերիկյան դոլար +$ ամն դոլար +som ուզբեկական սոմ +₩ վոն +¥ յեն \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/measurement_dates.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/measurement_dates.tsv new file mode 100644 index 000000000..da54d562e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/measurement_dates.tsv @@ -0,0 +1,4 @@ +թ. թվական +թթ. թվականներ +դ. դար +դդ. դարեր \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/measurements.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/measurements.tsv new file mode 100644 index 000000000..a2b355292 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/measurements.tsv @@ -0,0 +1,48 @@ +°F ֆարենհայթ +°C ցելսիուս +K կելվին +կմ կիլոմետր +կմ/ժ կիլոմետր ժամ +մ մետր +մ/ժ մետր ժամ +սմ սանտիմետր +մմ միլիմետր +հա հեկտար +մղն մղոն +մ² քառակուսի մետր +մ² մետր քառակուսի +կմ² քառակուսի կիլոմետր +կմ² կիլոմետր քառակուսի +% տոկոս +% տոկոսադրույք +Հց հերց +կՎտ կիլովատտ +կՎտ կիլո վատտ +կՎ/Ժ կիլովատտ ժամ +կՎ/ժ կիլո վատտ ժամ +Վտ/ժ վատտ ժամ +Վտ վատտ +ձ.ու. ձիաուժ +մգ միլիգրամ +կգ կիլոգրամ +Վ վոլտ +ժ ժամ +վ վայրկյան +ր րոպե +մ³ խորանարդ մետր +գ գրամ +տ տոննա +մբ մեգաբայթ +կբ կիլոբայթ +գբ գիգաբայթ +գբ գեգաբայթ +տբ տերաբայթ +տբ տեռաբայթ +կՎ կիլո վոլտ +մՎ մեգա վոլտ +Ա ամպեր +մԱ միլի ամպեր +մվ միլի վայրկյան +դմ դեցիմետր +սմ² քառակուսի սանտիմետր +սմ² քառակուսի սանտիմետր \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/digit.tsv new file mode 100644 index 000000000..4a38e1c3b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/digit.tsv @@ -0,0 +1,11 @@ +մեկ 1 +երկու 2 +երկուս 2 +երեք 3 +չորս 4 +հինգ 5 +վեց 6 +յոթ 7 +ութ 8 +ինը 9 +ինն 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/numbers/digits_no_one.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/digits_no_one.tsv new file mode 100644 index 000000000..71c2e6fa0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/digits_no_one.tsv @@ -0,0 +1,10 @@ +երկու 2 +երկուս 2 +երեք 3 +չորս 4 +հինգ 5 +վեց 6 +յոթ 7 +ութ 8 +ինը 9 +ինն 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/numbers/ties.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/ties.tsv new file mode 100644 index 000000000..514ac8135 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/numbers/ties.tsv @@ -0,0 +1,10 @@ +տասը 1 +տասն 1 +քսան 2 +երեսուն 3 +քառասուն 4 +հիսուն 5 +վաթսուն 6 +յոթանասուն 7 +ութսուն 8 +իննսուն 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/ordinals/__init__.py b/nemo_text_processing/inverse_text_normalization/hy/data/ordinals/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/ordinals/digit.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/ordinals/digit.tsv new file mode 100644 index 000000000..947449eb4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/ordinals/digit.tsv @@ -0,0 +1,10 @@ +զրոերորդ զրո +առաջին մեկ +երկրորդ երկու +երրորդ երեք +չորրորդ չորս +հինգերորդ հինգ +վեցերորդ վեց +յոթերորդ յոթ +ութերորդ ութ +իններորդ ինը \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/time/__init__.py b/nemo_text_processing/inverse_text_normalization/hy/data/time/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/time/hours.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/time/hours.tsv new file mode 100644 index 000000000..5cb60cada --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/time/hours.tsv @@ -0,0 +1,29 @@ +զրո 0 +մեկ 1 +երկու 2 +երկուս 2 +երեք 3 +չորս 4 +հինգ 5 +վեց 6 +յոթ 7 +ութ 8 +ինը 9 +տաս 10 +տասն 10 +տասնմեկ 11 +տասներկու 12 +տասներկուս 12 +տասներեք 13 +տասնչորս 14 +տասնհինգ 15 +տասնվեց 16 +տասնյոթ 17 +տասնութ 18 +տասնինը 19 +քսան 20 +քսանմեկ 21 +քսաներկու 22 +քսաներկուս 22 +քսաներեք 23 +քսանչորս 24 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/time/minute_to.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/time/minute_to.tsv new file mode 100644 index 000000000..edab4d5b0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/time/minute_to.tsv @@ -0,0 +1,59 @@ +1 59 +2 58 +3 57 +4 56 +5 55 +6 54 +7 53 +8 52 +9 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 9 +52 8 +53 7 +54 6 +55 5 +56 4 +57 3 +58 2 +59 1 diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/time/minutes.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/time/minutes.tsv new file mode 100644 index 000000000..0f31d43bc --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/time/minutes.tsv @@ -0,0 +1,65 @@ +մեկ 01 +երկու 02 +երկուս 02 +երեք 03 +չորս 04 +հինգ 05 +վեց 06 +յոթ 07 +ութ 08 +ինը 09 +տասը 10 +տասնմեկ 11 +տասներկու 12 +տասներկուս 12 +տասներեք 13 +տասնչորս 14 +տասնհինգ 15 +տասնվեց 16 +տասնյոթ 17 +տասնութ 18 +տասնինը 19 +քսան 20 +քսանմեկ 21 +քսաներկու 22 +քսաներկուս 22 +քսաներեք 23 +քսանչորս 24 +քսանհինգ 25 +քսանվեց 26 +քսանյոթ 27 +քսանութ 28 +քսանինը 29 +երեսուն 30 +երեսունմեկ 31 +երեսուներկու 32 +երեսուներկուս 32 +երեսուներեք 33 +երեսունչորս 34 +երեսունհինգ 35 +երեսունվեց 36 +երեսունյոթ 37 +երեսունութ 38 +երեսունինը 39 +քառասուն 40 +քառասունմեկ 41 +քառասուներկու 42 +քառասուներկուս 42 +քառասուներեք 43 +քառասունչորս 44 +քառասունհինգ 45 +քառասունվեց 46 +քառասունյոթ 47 +քառասունութ 48 +քառասունինը 49 +հիսուն 50 +հիսունմեկ 51 +հիսուներկու 52 +հիսուներկուս 52 +հիսուներեք 53 +հիսունչորս 54 +հիսունհինգ 55 +հիսունվեց 56 +հիսունյոթ 57 +հիսունութ 58 +հիսունինը 59 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/time/minutes_to.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/time/minutes_to.tsv new file mode 100644 index 000000000..0837e850a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/time/minutes_to.tsv @@ -0,0 +1,59 @@ +01 59 +02 58 +03 57 +04 56 +05 55 +06 54 +07 53 +08 52 +09 51 +10 50 +11 49 +12 48 +13 47 +14 46 +15 45 +16 44 +17 43 +18 42 +19 41 +20 40 +21 39 +22 38 +23 37 +24 36 +25 35 +26 34 +27 33 +28 32 +29 31 +30 30 +31 29 +32 28 +33 27 +34 26 +35 25 +36 24 +37 23 +38 22 +39 21 +40 20 +41 19 +42 18 +43 17 +44 16 +45 15 +46 14 +47 13 +48 12 +49 11 +50 10 +51 09 +52 08 +53 07 +54 06 +55 05 +56 04 +57 03 +58 02 +59 01 diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/time/to_hour.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/time/to_hour.tsv new file mode 100644 index 000000000..a56219579 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/time/to_hour.tsv @@ -0,0 +1,25 @@ +1 0 +2 1 +3 2 +4 3 +5 4 +6 5 +7 6 +8 7 +9 8 +10 9 +11 10 +12 11 +13 12 +14 13 +15 14 +16 15 +17 16 +18 17 +19 18 +20 19 +21 20 +22 21 +23 22 +24 23 +0 23 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hy/data/whitelist.tsv b/nemo_text_processing/inverse_text_normalization/hy/data/whitelist.tsv new file mode 100644 index 000000000..0e400c28b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/data/whitelist.tsv @@ -0,0 +1,7 @@ +ս.թ. սույն թվականի +մ.թ.ա. մեր թվարկությունից առաջ +մ.թ. մեր թվարկություն +Ք.ա. քրիստոսից առաջ +Ք.հ. քրիստոսից հետո +Ք.ծ.ա. քրիստոսի ծնունդից առաջ +Ք.ծ.հ. քրիստոսից ծնունդից հետո diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/cardinal.py new file mode 100644 index 000000000..1a74ddc3e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/cardinal.py @@ -0,0 +1,101 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. իննսունյոթ -> cardinal { integer: "97" } } + """ + + def __init__(self): + super().__init__(name="cardinal", kind="classify") + + zero = pynini.string_map([("զրո", "0")]) + digit = (pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) + ( + pynini.closure(pynutil.delete("ն") | pynutil.delete("ի") | pynutil.delete("ին"), 0, 1) + ) + digits_no_one = pynini.string_file(get_abs_path("data/numbers/digits_no_one.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + ( + pynini.closure(pynutil.delete("ն") | pynutil.delete("ի") | pynutil.delete("ին"), 0, 1) + ) + graph_digit = digit | pynutil.insert("0") + + graph_ties = graph_ties | pynutil.insert("0") + graph_two_digit_nums = graph_ties + graph_digit + + hundred = pynini.accep("հարյուր") + graph_hundred = pynini.cross("հարյուր", "1") + + graph_hundreds_first_digit = graph_hundred | (digits_no_one + delete_space + pynutil.delete(hundred)) + graph_hundreds = ( + (graph_hundreds_first_digit + delete_space | pynutil.insert("0", weight=0.1)) + + delete_space + + graph_two_digit_nums + ) + + self.graph_hundred_component_at_least_one_none_zero_digit = ( + graph_hundreds @ (pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)).optimize() + ) + + graph_one_thousand = pynini.cross("հազար", "1") + graph_many_thousand = graph_hundreds + delete_space + pynutil.delete("հազար") + graph_thousands = ( + (graph_one_thousand | graph_many_thousand | pynutil.insert("000", weight=0.000000001)) + + delete_space + + graph_hundreds + ) + + millions = pynini.accep("միլիոն") + graph_millions = ( + ((graph_hundreds + delete_space + pynutil.delete(millions)) | pynutil.insert("000", weight=0.1)) + + delete_space + + graph_thousands + ) + + billions = pynini.accep("միլիարդ") + graph_billions = ( + (graph_hundreds + delete_space + pynutil.delete(billions) + delete_space) + | pynutil.insert("000", weight=0.1) + ) + graph_millions + + trillions = pynini.accep("տրիլիոն") + graph_trillions = ( + (graph_hundreds + delete_space + pynutil.delete(trillions) + delete_space) + | pynutil.insert("000", weight=0.1) + ) + graph_billions + + graph = graph_trillions | zero + + delete_leading_zeroes = pynutil.delete(pynini.closure("0")) + stop_at_non_zero = pynini.difference(NEMO_DIGIT, "0") + rest_of_cardinal = pynini.closure(NEMO_DIGIT) + + clean_cardinal = delete_leading_zeroes + stop_at_non_zero + rest_of_cardinal + clean_cardinal = clean_cardinal | "0" + + graph = graph @ clean_cardinal + self.graph_no_exception = graph.optimize() + + final_graph = pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/decimal.py new file mode 100644 index 000000000..be52779f5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/decimal.py @@ -0,0 +1,117 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + MIN_NEG_WEIGHT, + NEMO_DIGIT, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + delete_extra_space, + delete_space, +) + + +def get_quantity( + decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike', input_case: str = INPUT_LOWER_CASED +) -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. հինգ միլիոն -> tokens { decimal { integer_part: "5" quantity: "միլիոն" } } + e.g. հինգ ամբողջ յոթ միլիարդ -> tokens { decimal { integer_part: "5" fractional_part: "7" quantity: "միլիարդ" } } + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + input_case: accepting either "lower_cased" or "cased" input. + (input_case is not necessary everything is made for lower_cased input) + TODO add case input support + + """ + numbers = cardinal_up_to_hundred @ ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + + suffix = pynini.union("միլիոն", "միլիարդ", "տրիլիոն") + + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + delete_extra_space + + pynutil.insert("quantity: \"") + + suffix + + pynutil.insert("\"") + ) + res |= decimal + delete_extra_space + pynutil.insert("quantity: \"") + (suffix | "հազար") + pynutil.insert("\"") + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal + e.g. հիսուն ու կես տրիլիոն -> decimal { integer_part: "50" fractional_part: "5" quantity: "տրիլիոն" } + e.g. մեկ միլիարդ -> decimal { integer_part: "1" quantity: "միլիարդ" } + Args: + cardinal: CardinalFst + input_case: accepting either "lower_cased" or "cased" input. + TODO add cased input support + """ + + def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="decimal", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + + graph_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) | pynini.string_map( + [("զրո", "0"), ("կես", "5")] + ) + + graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal + self.only_decimal = graph_decimal.optimize() + + point_first = pynutil.delete("ամբողջ") + point_second = pynutil.delete("ու") + + graph_fractional = pynutil.insert("fractional_part: \"") + graph_decimal + pynutil.insert("\"") + graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + final_graph_wo_sign = ( + pynini.closure((graph_integer | pynini.string_map(["", "0"])) + delete_extra_space, 0, 1) + + (point_first | point_second) + + delete_extra_space + + graph_fractional + ) + final_graph = final_graph_wo_sign + + self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( + final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit, input_case=input_case + ) + + self.final_graph_wo_negative |= pynutil.add_weight( + pynini.compose(TO_LOWER + NEMO_SIGMA, self.final_graph_wo_negative).optimize(), MIN_NEG_WEIGHT + ) + + quantity_graph = get_quantity( + final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit, input_case=input_case + ) + final_graph |= quantity_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/fraction.py new file mode 100644 index 000000000..a7eba809f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/fraction.py @@ -0,0 +1,44 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + e.g. երկու երրորդ -> tokens { fraction { numerator: "2" denominator: "3" } } + + Args: + input_case: accepting either "lower_cased" or "cased" input. + (input_case is not necessary everything is made for lower_cased input) + TODO add cased input support + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="fraction", kind="classify") + cardinal_graph = cardinal.graph_no_exception + quarter = pynini.string_map([("քառորդ", "4")]) + ordinal_graph = ordinal.graph | quarter + + numerator = pynutil.insert("numerator: \"") + cardinal_graph + pynutil.insert("\"") + denominator = pynutil.insert(" denominator: \"") + ordinal_graph + pynutil.insert("\"") + + final_graph = numerator + delete_space + denominator + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/measure.py new file mode 100644 index 000000000..a0814c582 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/measure.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + convert_space, + delete_extra_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. իննսունյոթ հերց -> measure { cardinal { integer: "97" } units: "Հց" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + input_case: accepting either "lower_cased" or "cased" input. + (input_case is not necessary everything is made for lower_cased input) + TODO add cased input support + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="measure", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + from_to = pynini.string_map([("ից", "")]) + cardinal_graph += pynutil.insert("") | from_to + + casing_graph = pynini.closure(TO_LOWER | NEMO_SIGMA).optimize() + + graph_measurements_unit = pynini.string_file(get_abs_path("data/measurements.tsv")) + ( + pynutil.insert("") | pynutil.insert("ում") | pynutil.insert("ից") + ) + graph_measurements_unit = pynini.invert(graph_measurements_unit) + graph_measurements_unit = pynini.compose(casing_graph, graph_measurements_unit).optimize() + + measurements_unit = convert_space(graph_measurements_unit) + + graph_measurements_dates_unit = pynini.string_file(get_abs_path("data/measurement_dates.tsv")) + + graph_measurements_dates_unit = pynini.invert(graph_measurements_dates_unit) + graph_measurements_dates_unit = pynini.compose(casing_graph, graph_measurements_dates_unit).optimize() + + measurements_dates_unit = convert_space(graph_measurements_dates_unit) + + measurements_unit = pynutil.insert("units: \"") + measurements_unit + pynutil.insert("\"") + + measurements_dates_unit = pynutil.insert("units: \"") + measurements_dates_unit + pynutil.insert("\"") + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" }") + + delete_extra_space + + measurements_unit + ) + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + measurements_unit + ) + subgraph_cardinal_dates = ( + (measurements_dates_unit + delete_extra_space | pynutil.insert("")) + + pynutil.insert("cardinal { ") + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + measurements_dates_unit + ) + subgraph_cardinal_dates |= ( + (measurements_dates_unit + delete_extra_space | pynutil.insert("")) + + pynutil.insert("cardinal { ") + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert('-') + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + delete_extra_space + + measurements_dates_unit + ) + + final_graph = subgraph_decimal | subgraph_cardinal | subgraph_cardinal_dates + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/money.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/money.py new file mode 100644 index 000000000..97b4d464c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/money.py @@ -0,0 +1,64 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_SIGMA, + GraphFst, + convert_space, + delete_extra_space, +) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money + e.g. քսան հազար դրամ -> tokens { money { integer_part: "20000" currency: "֏" } } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + input_case: accepting either "lower_cased" or "cased" input. + (input_case is not necessary everything is made for lower_cased input) + TODO add cased input support + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="money", kind="classify") + # quantity, integer_part, fractional_part, currency + + cardinal_graph = cardinal.graph_no_exception + graph_decimal_final = decimal.final_graph_wo_negative + unit = pynini.string_file(get_abs_path("data/currency.tsv")) + unit_singular = pynini.invert(unit) + + graph_unit_singular = pynutil.insert("currency: \"") + convert_space(unit_singular) + pynutil.insert("\"") + + graph_integer = ( + pynutil.insert("integer_part: \"") + + (NEMO_SIGMA @ cardinal_graph) + + pynutil.insert("\"") + + delete_extra_space + + graph_unit_singular + ) + graph_decimal = graph_decimal_final + delete_extra_space + graph_unit_singular + final_graph = graph_integer | graph_decimal + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/ordinal.py new file mode 100644 index 000000000..381480270 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/ordinal.py @@ -0,0 +1,55 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_CHAR, + GraphFst, + capitalized_input_graph, +) + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + e.g. հիսունյոթերորդ -> tokens { ordinal { integer: "57" } } + + Args: + cardinal: CardinalFst + input_case: accepting either "lower_cased" or "cased" input. + (input_case is not necessary everything is made for lower_cased input) + TODO add cased input support + """ + + def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="ordinal", kind="classify") + + cardinal_graph = cardinal.graph_no_exception + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")) + graph = pynini.closure(NEMO_CHAR) + pynini.union(graph_digit, pynini.cross("երորդ", "")) + + self.graph = pynini.compose(graph, cardinal_graph).optimize() + + if input_case == INPUT_CASED: + self.graph = capitalized_input_graph(self.graph) + + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/punctuation.py new file mode 100644 index 000000000..a6c78a7c1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/punctuation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. , -> tokens { name: "," } + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~,։՜՝" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/time.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/time.py new file mode 100644 index 000000000..1608cb6a0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/time.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time + e.g. տասներկուսն անց հինգ -> time { hours: "12" minutes: "05" } + e.g. հինգին տասնհինգ պակաս -> time { hours: "04" minutes: "45" } + e.g. տասներեք անց կես -> time { hours: "12" minutes: "30" } + """ + + def __init__(self): + super().__init__(name="time", kind="classify") + graph_oclock = pynutil.delete("անց") + + graph_demi = pynini.cross("կես", "30") + + graph_fractions = graph_demi + + graph_hours = pynini.string_file(get_abs_path("data/time/hours.tsv")) + ( + pynini.closure(pynutil.delete("ն") | pynutil.delete("ին"), 0, 1) + ) + graph_minutes = pynini.string_file(get_abs_path("data/time/minutes.tsv")) + ( + pynini.closure(pynutil.delete("ն") | pynutil.delete("ին"), 0, 1) + ) + graph_hours_to = pynini.string_file(get_abs_path("data/time/to_hour.tsv")) + graph_minutes_to = pynini.string_file(get_abs_path("data/time/minutes_to.tsv")) + graph_to = pynutil.delete("պակաս") + + graph_hours_component = pynutil.insert("hours: \"") + graph_hours + pynutil.insert("\"") + + graph_minutes_component = ( + pynutil.insert(" minutes: \"") + pynini.union(graph_minutes, graph_fractions) + pynutil.insert("\"") + ) + graph_minutes_component = delete_space + graph_minutes_component + + graph_time_standard = ( + graph_hours_component + delete_space + graph_oclock + pynini.closure(graph_minutes_component, 0, 1) + ) + + graph_hours_to_component = graph_hours + pynutil.delete('ին') + graph_hours_to_component @= graph_hours_to + graph_hours_to_component = pynutil.insert("hours: \"") + graph_hours_to_component + pynutil.insert("\"") + + graph_minutes_to_component = graph_minutes + graph_minutes_to_component @= graph_minutes_to + graph_minutes_to_component = pynutil.insert(" minutes: \"") + graph_minutes_to_component + pynutil.insert("\"") + + graph_time_to = graph_hours_to_component + delete_space + graph_minutes_to_component + delete_space + graph_to + + graph_time_no_suffix = graph_time_standard | graph_time_to + + final_graph = graph_time_no_suffix + + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..2f778e77c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/tokenize_and_classify.py @@ -0,0 +1,114 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.hy.taggers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__( + self, + cache_dir: str = None, + whitelist: str = None, + overwrite_cache: bool = False, + input_case: str = INPUT_LOWER_CASED, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"_hy_itn_{input_case}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + ordinal_graph = ordinal.fst + + fraction = FractionFst(cardinal, ordinal) + fraction_graph = fraction.fst + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst + word_graph = WordFst().fst + time_graph = TimeFst().fst + money_graph = MoneyFst(cardinal, decimal).fst + punct_graph = PunctuationFst().fst + whitelist_graph = WhiteListFst().fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.05) + | pynutil.add_weight(decimal_graph, 1.08) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.09) + | pynutil.add_weight(money_graph, 1.07) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/whitelist.py new file mode 100644 index 000000000..212fe80f2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/whitelist.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.utils import get_abs_path +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + e.g. մեր թվարկությունից առաջ -> tokens { name: "մ.թ.ա" } + This class has highest priority among all classifier grammars. + Whitelisted tokens are defined and loaded from "data/whitelist.tsv" (unless input_file specified). + """ + + def __init__(self): + super().__init__(name="whitelist", kind="classify") + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert() + graph = pynutil.insert("name: \"") + convert_space(whitelist) + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/taggers/word.py b/nemo_text_processing/inverse_text_normalization/hy/taggers/word.py new file mode 100644 index 000000000..74a356072 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/taggers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. արթնանալ -> tokens { name: "արթնանալ" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/utils.py b/nemo_text_processing/inverse_text_normalization/hy/utils.py new file mode 100644 index 000000000..1f1349115 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/utils.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/cardinal.py new file mode 100644 index 000000000..d80e41287 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/cardinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. cardinal { integer: "97" } -> 97 + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/decimal.py new file mode 100644 index 000000000..b5caee6a8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/decimal.py @@ -0,0 +1,58 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { integer_part: "12" fractional_part: "5" quantity: "միլիարդ" } -> 12.5 միլիարդ + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_integer = pynini.closure(integer + delete_space, 0, 1) + fractional = ( + pynutil.insert(".") + + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_fractional = pynini.closure(fractional + delete_space, 0, 1) + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(pynutil.insert(NEMO_SPACE) + quantity + delete_space, 0, 1) + graph = optional_integer + optional_fractional + optional_quantity + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/fraction.py new file mode 100644 index 000000000..fb96f95d5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/fraction.py @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { numerator: "2" denominator: "3" } } -> 2/3 + + """ + + def __init__(self): + super().__init__(name="fraction", kind="verbalize") + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + denominator = ( + pynutil.insert('/') + + pynutil.delete("denominator: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph = (numerator + delete_space + denominator).optimize() + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/measure.py new file mode 100644 index 000000000..68401691f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/measure.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SPACE, GraphFst, delete_space + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "59" } units: "Հց" } -> 59 Հց + + Args: + decimal: DecimalFst + cardinal: CardinalFst + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst): + super().__init__(name="measure", kind="verbalize") + unit = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + + pynutil.delete("\"") + + delete_space + ) + graph_decimal = ( + pynutil.delete("decimal {") + delete_space + decimal.numbers + delete_space + pynutil.delete("}") + ) + graph_cardinal_first = ( + pynutil.delete("cardinal {") + delete_space + cardinal.numbers + delete_space + pynutil.delete("} ") + ) + + graph_cardinal_two = ( + pynutil.delete("cardinal {") + + pynutil.delete(" integer: \"") + + delete_space + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("} ") + ) + + graph_first = (graph_cardinal_first | graph_decimal) + delete_space + pynutil.insert(" ") + unit + graph_second = graph_cardinal_two + delete_space + pynutil.insert(" ") + unit + graph = graph_first | graph_second + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/money.py new file mode 100644 index 000000000..3af78c955 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/money.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "20000" currency: "֏" } -> 20000 ֏ + + Args: + decimal: DecimalFst + """ + + def __init__(self, decimal: GraphFst): + super().__init__(name="money", kind="verbalize") + unit = ( + pynutil.delete("currency:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + + pynutil.delete("\"") + ) + graph = decimal.numbers + delete_space + insert_space + unit + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/ordinal.py new file mode 100644 index 000000000..b0d4e52cc --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/ordinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + tokens { ordinal { integer: "3" } } -> 3-րդ + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + convert_one = pynini.cross("[BOS]1", "[BOS]1-ին") + convert_rest = pynutil.insert("-րդ", weight=0.01) + + suffix = pynini.cdrewrite( + convert_rest | convert_one, + "", + "[EOS]", + NEMO_SIGMA, + ) + graph = graph @ suffix + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/time.py new file mode 100644 index 000000000..7edbc043a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/time.py @@ -0,0 +1,53 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + tokens { time { hours: "4" minutes: "45" } } -> 04:45 + tokens { time { hours: "1" minutes: "12" } } -> 01:12 + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + graph = ( + hour @ add_leading_zero_to_double_digit + + delete_space + + pynutil.insert(":") + + (minute @ add_leading_zero_to_double_digit) + ) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize.py new file mode 100644 index 000000000..bf3635a40 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize.py @@ -0,0 +1,51 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class VerbalizeFst(GraphFst): + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + ordinal_graph = OrdinalFst().fst + decimal = DecimalFst() + decimal_graph = decimal.fst + fraction = FractionFst() + fraction_graph = fraction.fst + measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst + money_graph = MoneyFst(decimal=decimal).fst + time_graph = TimeFst().fst + whitelist_graph = WhiteListFst().fst + graph = ( + time_graph + | measure_graph + | fraction_graph + | money_graph + | ordinal_graph + | decimal_graph + | cardinal_graph + | whitelist_graph + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize_final.py new file mode 100644 index 000000000..39eb3a15f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/verbalize_final.py @@ -0,0 +1,45 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.hy.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "նա" } tokens { cardinal { integer: "23" } } tokens { name: "տարեկան" } tokens { name: "է" } -> նա 23 տարեկան է + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/whitelist.py new file mode 100644 index 000000000..1aa132328 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/whitelist.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "մ.թ.ա" } -> մ.թ.ա + """ + + def __init__(self): + super().__init__(name="whitelist", kind="verbalize") + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/hy/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/word.py new file mode 100644 index 000000000..a6887e528 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hy/verbalizers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens + e.g. tokens { name: "արթնանալ" } -> արթնանալ + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index 72c7f43e1..9a6fcc64c 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -101,7 +101,7 @@ def __init__( from nemo_text_processing.inverse_text_normalization.ar.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) - elif lang == 'es_en': # Arabic + elif lang == 'es_en': # Spanish-English code-switch from nemo_text_processing.inverse_text_normalization.es_en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.es_en.verbalizers.verbalize_final import ( VerbalizeFinalFst, @@ -111,6 +111,43 @@ def __init__( from nemo_text_processing.inverse_text_normalization.zh.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) + elif lang == 'mr': # Marathi + from nemo_text_processing.inverse_text_normalization.mr.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.mr.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + elif lang == 'hi': # Hindi + from nemo_text_processing.inverse_text_normalization.hi.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + elif lang == 'hi_en': # Hindi-English code-switch + from nemo_text_processing.inverse_text_normalization.hi_en.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + elif lang == 'hy': + from nemo_text_processing.inverse_text_normalization.hy.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.hy.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + elif lang == 'ja': # Japanese + from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + elif lang == 'he': # Japanese + from nemo_text_processing.inverse_text_normalization.he.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.he.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + elif lang == 'ko': # Korean + from nemo_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) + else: + raise NotImplementedError(f"Language {lang} has not been supported yet.") self.tagger = ClassifyFst( cache_dir=cache_dir, whitelist=whitelist, overwrite_cache=overwrite_cache, input_case=input_case @@ -155,7 +192,26 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en', 'zh'], + choices=[ + 'en', + 'de', + 'es', + 'pt', + 'ru', + 'fr', + 'sv', + 'vi', + 'ar', + 'es_en', + 'hi_en', + 'zh', + 'he', + 'hi', + 'hy', + 'mr', + 'ja', + 'ko', + ], default="en", type=str, ) diff --git a/nemo_text_processing/inverse_text_normalization/ja/__init__.py b/nemo_text_processing/inverse_text_normalization/ja/__init__.py new file mode 100644 index 000000000..ddf965368 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/__init__.py b/nemo_text_processing/inverse_text_normalization/ja/data/__init__.py new file mode 100644 index 000000000..ddf965368 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/char/__init__.py b/nemo_text_processing/inverse_text_normalization/ja/data/char/__init__.py new file mode 100644 index 000000000..a1cf281f0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/char/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/char/charset_extension.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/char/charset_extension.tsv new file mode 100644 index 000000000..de3ac0a83 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/char/charset_extension.tsv @@ -0,0 +1,6 @@ +吶 +诶 +屌 +囧 +飚 +屄 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/char/charset_national_standard_2013_8105.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/char/charset_national_standard_2013_8105.tsv new file mode 100644 index 000000000..06b46c630 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/char/charset_national_standard_2013_8105.tsv @@ -0,0 +1,8105 @@ +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丏 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丢 +两 +严 +丧 +个 +丫 +中 +丰 +串 +临 +丸 +丹 +为 +主 +丽 +举 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乸 +乾 +了 +予 +争 +事 +二 +亍 +于 +亏 +云 +互 +亓 +五 +井 +亘 +亚 +些 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +亶 +亸 +亹 +人 +亿 +什 +仁 +仂 +仃 +仄 +仅 +仆 +仇 +仉 +今 +介 +仍 +从 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +仡 +代 +令 +以 +仨 +仪 +仫 +们 +仰 +仲 +仳 +仵 +件 +价 +任 +份 +仿 +企 +伈 +伉 +伊 +伋 +伍 +伎 +伏 +伐 +休 +众 +优 +伙 +会 +伛 +伞 +伟 +传 +伢 +伣 +伤 +伥 +伦 +伧 +伪 +伫 +伭 +伯 +估 +伲 +伴 +伶 +伸 +伺 +似 +伽 +伾 +佁 +佃 +但 +位 +低 +住 +佐 +佑 +体 +何 +佖 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +佴 +佶 +佸 +佺 +佻 +佼 +佽 +佾 +使 +侁 +侂 +侃 +侄 +侈 +侉 +例 +侍 +侏 +侑 +侔 +侗 +侘 +供 +依 +侠 +侣 +侥 +侦 +侧 +侨 +侩 +侪 +侬 +侮 +侯 +侴 +侵 +侹 +便 +促 +俄 +俅 +俊 +俍 +俎 +俏 +俐 +俑 +俗 +俘 +俙 +俚 +俜 +保 +俞 +俟 +信 +俣 +俦 +俨 +俩 +俪 +俫 +俭 +修 +俯 +俱 +俳 +俵 +俶 +俸 +俺 +俾 +倌 +倍 +倏 +倒 +倓 +倔 +倕 +倘 +候 +倚 +倜 +倞 +借 +倡 +倥 +倦 +倧 +倨 +倩 +倪 +倬 +倭 +倮 +倴 +债 +倻 +值 +倾 +偁 +偃 +假 +偈 +偌 +偎 +偏 +偓 +偕 +做 +停 +偡 +健 +偬 +偭 +偰 +偲 +偶 +偷 +偻 +偾 +偿 +傀 +傃 +傅 +傈 +傉 +傍 +傒 +傕 +傣 +傥 +傧 +储 +傩 +催 +傲 +傺 +傻 +僇 +僎 +像 +僔 +僖 +僚 +僦 +僧 +僬 +僭 +僮 +僰 +僳 +僵 +僻 +儆 +儇 +儋 +儒 +儡 +儦 +儳 +儴 +儿 +兀 +允 +元 +兄 +充 +兆 +先 +光 +克 +免 +兑 +兔 +兕 +兖 +党 +兜 +兢 +入 +全 +八 +公 +六 +兮 +兰 +共 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +冁 +内 +冈 +冉 +册 +再 +冏 +冒 +冔 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冬 +冮 +冯 +冰 +冱 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +净 +凄 +准 +凇 +凉 +凋 +凌 +减 +凑 +凓 +凘 +凛 +凝 +几 +凡 +凤 +凫 +凭 +凯 +凰 +凳 +凶 +凸 +凹 +出 +击 +凼 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +刖 +列 +刘 +则 +刚 +创 +初 +删 +判 +刨 +利 +别 +刬 +刭 +刮 +到 +刳 +制 +刷 +券 +刹 +刺 +刻 +刽 +刿 +剀 +剁 +剂 +剃 +剅 +削 +剋 +剌 +前 +剐 +剑 +剔 +剕 +剖 +剜 +剞 +剟 +剡 +剥 +剧 +剩 +剪 +副 +割 +剽 +剿 +劁 +劂 +劄 +劈 +劐 +劓 +力 +劝 +办 +功 +加 +务 +劢 +劣 +动 +助 +努 +劫 +劬 +劭 +励 +劲 +劳 +劼 +劾 +势 +勃 +勇 +勉 +勋 +勍 +勐 +勒 +勔 +勖 +勘 +勚 +募 +勠 +勤 +勰 +勺 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匏 +匐 +匕 +化 +北 +匙 +匜 +匝 +匠 +匡 +匣 +匦 +匪 +匮 +匹 +区 +医 +匼 +匾 +匿 +十 +千 +卅 +升 +午 +卉 +半 +华 +协 +卑 +卒 +卓 +单 +卖 +南 +博 +卜 +卞 +卟 +占 +卡 +卢 +卣 +卤 +卦 +卧 +卫 +卬 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卺 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厍 +厕 +厖 +厘 +厚 +厝 +原 +厢 +厣 +厥 +厦 +厨 +厩 +厮 +去 +厾 +县 +叁 +参 +叆 +叇 +又 +叉 +及 +友 +双 +反 +发 +叔 +叕 +取 +受 +变 +叙 +叚 +叛 +叟 +叠 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +吣 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吲 +吴 +吵 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呃 +呆 +呇 +呈 +告 +呋 +呐 +呒 +呓 +呔 +呕 +呖 +呗 +员 +呙 +呛 +呜 +呢 +呣 +呤 +呦 +周 +呱 +呲 +味 +呵 +呶 +呷 +呸 +呻 +呼 +命 +咀 +咂 +咄 +咆 +咇 +咉 +咋 +和 +咍 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咙 +咚 +咛 +咝 +咡 +咣 +咤 +咥 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咯 +咱 +咳 +咴 +咸 +咺 +咻 +咽 +咿 +哀 +品 +哂 +哃 +哄 +哆 +哇 +哈 +哉 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哓 +哔 +哕 +哗 +哙 +哚 +哝 +哞 +哟 +哢 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哱 +哲 +哳 +哺 +哼 +哽 +哿 +唁 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唛 +唝 +唠 +唢 +唣 +唤 +唧 +唪 +唬 +售 +唯 +唰 +唱 +唳 +唵 +唷 +唼 +唾 +唿 +啁 +啃 +啄 +商 +啉 +啊 +啐 +啕 +啖 +啜 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啭 +啮 +啰 +啴 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喁 +喂 +喃 +善 +喆 +喇 +喈 +喉 +喊 +喋 +喏 +喑 +喔 +喘 +喙 +喜 +喝 +喟 +喤 +喧 +喱 +喳 +喵 +喷 +喹 +喻 +喽 +喾 +嗄 +嗅 +嗉 +嗌 +嗍 +嗐 +嗑 +嗒 +嗓 +嗔 +嗖 +嗜 +嗝 +嗞 +嗟 +嗡 +嗣 +嗤 +嗥 +嗦 +嗨 +嗪 +嗫 +嗬 +嗯 +嗲 +嗳 +嗵 +嗷 +嗽 +嗾 +嘀 +嘁 +嘈 +嘉 +嘌 +嘎 +嘏 +嘘 +嘚 +嘛 +嘞 +嘟 +嘡 +嘣 +嘤 +嘧 +嘬 +嘭 +嘱 +嘲 +嘴 +嘶 +嘹 +嘻 +嘿 +噀 +噂 +噇 +噌 +噍 +噎 +噔 +噗 +噘 +噙 +噜 +噢 +噤 +器 +噩 +噪 +噫 +噬 +噱 +噶 +噻 +噼 +嚄 +嚅 +嚆 +嚎 +嚏 +嚓 +嚚 +嚣 +嚭 +嚯 +嚷 +嚼 +囊 +囔 +囚 +四 +回 +囟 +因 +囡 +团 +囤 +囫 +园 +困 +囱 +围 +囵 +囷 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +圉 +圊 +圌 +圐 +圙 +圜 +土 +圢 +圣 +在 +圩 +圪 +圫 +圬 +圭 +圮 +圯 +地 +圲 +圳 +圹 +场 +圻 +圾 +址 +坂 +均 +坉 +坊 +坋 +坌 +坍 +坎 +坏 +坐 +坑 +坒 +块 +坚 +坛 +坜 +坝 +坞 +坟 +坠 +坡 +坤 +坥 +坦 +坨 +坩 +坪 +坫 +坬 +坭 +坯 +坰 +坳 +坷 +坻 +坼 +坽 +垂 +垃 +垄 +垆 +垈 +型 +垌 +垍 +垎 +垏 +垒 +垓 +垕 +垙 +垚 +垛 +垞 +垟 +垠 +垡 +垢 +垣 +垤 +垦 +垧 +垩 +垫 +垭 +垮 +垯 +垱 +垲 +垴 +垵 +垸 +垺 +垾 +垿 +埂 +埃 +埆 +埇 +埋 +埌 +城 +埏 +埒 +埔 +埕 +埗 +埘 +埙 +埚 +埝 +域 +埠 +埤 +埪 +埫 +埭 +埯 +埴 +埵 +埸 +培 +基 +埼 +埽 +堂 +堃 +堆 +堇 +堉 +堋 +堌 +堍 +堎 +堐 +堑 +堕 +堙 +堞 +堠 +堡 +堤 +堧 +堨 +堪 +堰 +堲 +堵 +堼 +堽 +堾 +塄 +塅 +塆 +塌 +塍 +塑 +塔 +塘 +塝 +塞 +塥 +填 +塬 +塱 +塾 +墀 +墁 +境 +墅 +墈 +墉 +墐 +墒 +墓 +墕 +墘 +墙 +墚 +增 +墟 +墡 +墣 +墦 +墨 +墩 +墼 +壁 +壅 +壑 +壕 +壤 +士 +壬 +壮 +声 +壳 +壶 +壸 +壹 +处 +备 +复 +夏 +夐 +夔 +夕 +外 +夙 +多 +夜 +够 +夤 +夥 +大 +天 +太 +夫 +夬 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夼 +奁 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +契 +奓 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奡 +奢 +奥 +奭 +女 +奴 +奶 +奸 +她 +好 +妁 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妗 +妘 +妙 +妞 +妣 +妤 +妥 +妧 +妨 +妩 +妪 +妫 +妭 +妮 +妯 +妲 +妹 +妻 +妾 +姆 +姈 +姊 +始 +姐 +姑 +姒 +姓 +委 +姗 +姘 +姚 +姜 +姝 +姞 +姣 +姤 +姥 +姨 +姬 +姮 +姱 +姶 +姹 +姻 +姽 +姿 +娀 +威 +娃 +娄 +娅 +娆 +娇 +娈 +娉 +娌 +娑 +娓 +娘 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娵 +娶 +娼 +婀 +婆 +婉 +婊 +婌 +婍 +婕 +婘 +婚 +婞 +婠 +婢 +婤 +婧 +婪 +婫 +婳 +婴 +婵 +婶 +婷 +婺 +婻 +婼 +婿 +媂 +媄 +媆 +媒 +媓 +媖 +媚 +媛 +媞 +媪 +媭 +媱 +媲 +媳 +媵 +媸 +媾 +嫁 +嫂 +嫄 +嫉 +嫌 +嫒 +嫔 +嫕 +嫖 +嫘 +嫚 +嫜 +嫠 +嫡 +嫣 +嫦 +嫩 +嫪 +嫫 +嫭 +嫱 +嫽 +嬉 +嬖 +嬗 +嬛 +嬥 +嬬 +嬴 +嬷 +嬿 +孀 +孅 +子 +孑 +孓 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +孥 +学 +孩 +孪 +孬 +孰 +孱 +孳 +孵 +孺 +孽 +宁 +它 +宄 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +宠 +审 +客 +宣 +室 +宥 +宦 +宧 +宪 +宫 +宬 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寁 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寝 +寞 +察 +寡 +寤 +寥 +寨 +寮 +寰 +寸 +对 +寺 +寻 +导 +寿 +封 +射 +将 +尉 +尊 +小 +少 +尔 +尕 +尖 +尘 +尚 +尜 +尝 +尢 +尤 +尥 +尧 +尨 +尪 +尬 +就 +尴 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屃 +居 +屈 +屉 +届 +屋 +屎 +屏 +屐 +屑 +展 +屙 +属 +屠 +屡 +屣 +履 +屦 +屯 +山 +屹 +屺 +屼 +屾 +屿 +岁 +岂 +岈 +岊 +岌 +岍 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岜 +岞 +岠 +岢 +岣 +岨 +岩 +岫 +岬 +岭 +岱 +岳 +岵 +岷 +岸 +岽 +岿 +峁 +峂 +峃 +峄 +峋 +峒 +峗 +峘 +峙 +峛 +峡 +峣 +峤 +峥 +峦 +峧 +峨 +峪 +峭 +峰 +峱 +峻 +峿 +崀 +崁 +崂 +崃 +崄 +崆 +崇 +崌 +崎 +崒 +崔 +崖 +崚 +崛 +崞 +崟 +崡 +崤 +崦 +崧 +崩 +崭 +崮 +崴 +崶 +崽 +崾 +崿 +嵁 +嵅 +嵇 +嵊 +嵋 +嵌 +嵎 +嵖 +嵘 +嵚 +嵛 +嵝 +嵩 +嵫 +嵬 +嵯 +嵲 +嵴 +嶂 +嶅 +嶍 +嶒 +嶓 +嶙 +嶝 +嶟 +嶦 +嶲 +嶷 +巅 +巇 +巉 +巍 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +巯 +己 +已 +巳 +巴 +巷 +巽 +巾 +币 +市 +布 +帅 +帆 +师 +希 +帏 +帐 +帑 +帔 +帕 +帖 +帘 +帙 +帚 +帛 +帜 +帝 +帡 +带 +帧 +帨 +席 +帮 +帱 +帷 +常 +帻 +帼 +帽 +幂 +幄 +幅 +幌 +幔 +幕 +幖 +幛 +幞 +幡 +幢 +幪 +干 +平 +年 +并 +幸 +幺 +幻 +幼 +幽 +广 +庄 +庆 +庇 +床 +庋 +序 +庐 +庑 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +庤 +庥 +度 +座 +庭 +庱 +庳 +庵 +庶 +康 +庸 +庹 +庼 +庾 +廆 +廉 +廊 +廋 +廑 +廒 +廓 +廖 +廙 +廛 +廨 +廪 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弆 +弇 +弈 +弊 +弋 +式 +弑 +弓 +引 +弗 +弘 +弛 +弟 +张 +弢 +弥 +弦 +弧 +弨 +弩 +弭 +弯 +弱 +弶 +弸 +弹 +强 +弼 +彀 +归 +当 +录 +彖 +彗 +彘 +彝 +彟 +形 +彤 +彦 +彧 +彩 +彪 +彬 +彭 +彰 +影 +彳 +彷 +役 +彻 +彼 +往 +征 +徂 +径 +待 +徇 +很 +徉 +徊 +律 +徐 +徒 +徕 +得 +徘 +徙 +徛 +徜 +御 +徨 +循 +徭 +微 +徵 +德 +徼 +徽 +心 +必 +忆 +忉 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +忝 +忞 +忠 +忡 +忤 +忧 +忪 +快 +忭 +忮 +忱 +忳 +念 +忸 +忺 +忻 +忽 +忾 +忿 +怀 +态 +怂 +怃 +怄 +怅 +怆 +怊 +怍 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怛 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怩 +怪 +怫 +怯 +怵 +总 +怼 +怿 +恁 +恂 +恃 +恋 +恍 +恐 +恒 +恓 +恔 +恕 +恙 +恚 +恝 +恢 +恣 +恤 +恧 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恶 +恸 +恹 +恺 +恻 +恼 +恽 +恿 +悃 +悄 +悆 +悈 +悉 +悌 +悍 +悒 +悔 +悖 +悚 +悛 +悝 +悟 +悠 +悢 +患 +悦 +您 +悫 +悬 +悭 +悯 +悰 +悱 +悲 +悴 +悸 +悻 +悼 +情 +惆 +惇 +惊 +惋 +惎 +惑 +惔 +惕 +惘 +惙 +惚 +惛 +惜 +惝 +惟 +惠 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +想 +惴 +惶 +惹 +惺 +愀 +愁 +愃 +愆 +愈 +愉 +愍 +愎 +意 +愐 +愔 +愕 +愚 +感 +愠 +愣 +愤 +愦 +愧 +愫 +愭 +愿 +慆 +慈 +慊 +慌 +慎 +慑 +慕 +慝 +慢 +慥 +慧 +慨 +慬 +慭 +慰 +慵 +慷 +憋 +憎 +憔 +憕 +憙 +憧 +憨 +憩 +憬 +憭 +憷 +憺 +憾 +懂 +懈 +懊 +懋 +懑 +懒 +懔 +懦 +懵 +懿 +戆 +戈 +戊 +戋 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +戗 +战 +戚 +戛 +戟 +戡 +戢 +戣 +戤 +戥 +截 +戬 +戭 +戮 +戳 +戴 +户 +戽 +戾 +房 +所 +扁 +扂 +扃 +扅 +扆 +扇 +扈 +扉 +扊 +手 +才 +扎 +扑 +扒 +打 +扔 +托 +扛 +扞 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扳 +扶 +批 +扺 +扼 +扽 +找 +承 +技 +抃 +抄 +抉 +把 +抑 +抒 +抓 +抔 +投 +抖 +抗 +折 +抚 +抛 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +抻 +押 +抽 +抿 +拂 +拃 +拄 +担 +拆 +拇 +拈 +拉 +拊 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +招 +拜 +拟 +拢 +拣 +拤 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拶 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挓 +挖 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挦 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +捂 +捃 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捩 +捭 +据 +捯 +捶 +捷 +捺 +捻 +捽 +掀 +掂 +掇 +授 +掉 +掊 +掌 +掎 +掏 +掐 +排 +掖 +掘 +掞 +掠 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掭 +掮 +掰 +掳 +掴 +掷 +掸 +掺 +掼 +掾 +揄 +揆 +揉 +揍 +描 +提 +插 +揕 +揖 +揠 +握 +揣 +揩 +揪 +揭 +揳 +援 +揶 +揸 +揽 +揿 +搀 +搁 +搂 +搅 +搋 +搌 +搏 +搐 +搒 +搓 +搔 +搛 +搜 +搞 +搠 +搡 +搦 +搪 +搬 +搭 +搴 +携 +搽 +摁 +摄 +摅 +摆 +摇 +摈 +摊 +摏 +摒 +摔 +摘 +摛 +摞 +摧 +摩 +摭 +摴 +摸 +摹 +摽 +撂 +撄 +撅 +撇 +撑 +撒 +撕 +撖 +撙 +撞 +撤 +撩 +撬 +播 +撮 +撰 +撵 +撷 +撸 +撺 +撼 +擀 +擂 +擅 +操 +擎 +擐 +擒 +擘 +擞 +擢 +擤 +擦 +擿 +攀 +攉 +攒 +攘 +攥 +攫 +攮 +支 +收 +攸 +改 +攻 +攽 +放 +政 +故 +效 +敉 +敌 +敏 +救 +敔 +敕 +敖 +教 +敛 +敝 +敞 +敢 +散 +敦 +敩 +敫 +敬 +数 +敲 +整 +敷 +文 +斋 +斌 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斝 +斟 +斠 +斡 +斤 +斥 +斧 +斩 +斫 +断 +斯 +新 +斶 +方 +於 +施 +旁 +旃 +旄 +旅 +旆 +旋 +旌 +旎 +族 +旐 +旒 +旖 +旗 +旞 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旯 +旰 +旱 +旴 +旵 +时 +旷 +旸 +旺 +旻 +旿 +昀 +昂 +昃 +昄 +昆 +昇 +昈 +昉 +昊 +昌 +明 +昏 +昒 +易 +昔 +昕 +昙 +昝 +星 +映 +昡 +昣 +昤 +春 +昧 +昨 +昪 +昫 +昭 +是 +昱 +昳 +昴 +昵 +昶 +昺 +昼 +昽 +显 +晁 +晃 +晅 +晊 +晋 +晌 +晏 +晐 +晒 +晓 +晔 +晕 +晖 +晗 +晙 +晚 +晞 +晟 +晡 +晢 +晤 +晦 +晨 +晪 +晫 +普 +景 +晰 +晱 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暅 +暇 +暌 +暑 +暕 +暖 +暗 +暝 +暧 +暨 +暮 +暲 +暴 +暵 +暶 +暹 +暾 +暿 +曈 +曌 +曙 +曛 +曜 +曝 +曦 +曩 +曰 +曲 +曳 +更 +曷 +曹 +曼 +曾 +替 +最 +月 +有 +朋 +服 +朏 +朐 +朓 +朔 +朕 +朗 +望 +朝 +期 +朦 +木 +未 +末 +本 +札 +术 +朱 +朳 +朴 +朵 +朸 +机 +朽 +杀 +杂 +权 +杄 +杆 +杈 +杉 +杌 +李 +杏 +材 +村 +杓 +杕 +杖 +杙 +杜 +杞 +束 +杠 +条 +来 +杧 +杨 +杩 +杪 +杭 +杯 +杰 +杲 +杳 +杵 +杷 +杻 +杼 +松 +板 +极 +构 +枅 +枇 +枉 +枋 +枍 +析 +枕 +林 +枘 +枚 +果 +枝 +枞 +枢 +枣 +枥 +枧 +枨 +枪 +枫 +枭 +枯 +枰 +枲 +枳 +枵 +架 +枷 +枸 +枹 +柁 +柃 +柄 +柈 +柊 +柏 +某 +柑 +柒 +染 +柔 +柖 +柘 +柙 +柚 +柜 +柝 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柰 +柱 +柳 +柴 +柷 +柽 +柿 +栀 +栅 +标 +栈 +栉 +栊 +栋 +栌 +栎 +栏 +栐 +树 +栒 +栓 +栖 +栗 +栝 +栟 +校 +栩 +株 +栲 +栳 +栴 +样 +核 +根 +栻 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桄 +桅 +框 +案 +桉 +桊 +桌 +桎 +桐 +桑 +桓 +桔 +桕 +桠 +桡 +桢 +档 +桤 +桥 +桦 +桧 +桨 +桩 +桫 +桯 +桲 +桴 +桶 +桷 +桹 +梁 +梃 +梅 +梆 +梌 +梏 +梓 +梗 +梠 +梢 +梣 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梴 +梵 +梼 +梽 +梾 +梿 +检 +棁 +棂 +棉 +棋 +棍 +棐 +棒 +棓 +棕 +棘 +棚 +棠 +棣 +棤 +棨 +棪 +棫 +棬 +森 +棰 +棱 +棵 +棹 +棺 +棻 +棼 +棽 +椀 +椁 +椅 +椆 +椋 +植 +椎 +椐 +椑 +椒 +椓 +椟 +椠 +椤 +椪 +椭 +椰 +椴 +椸 +椹 +椽 +椿 +楂 +楒 +楔 +楗 +楙 +楚 +楝 +楞 +楠 +楣 +楦 +楩 +楪 +楫 +楮 +楯 +楷 +楸 +楹 +楼 +概 +榃 +榄 +榅 +榆 +榇 +榈 +榉 +榍 +榑 +榔 +榕 +榖 +榛 +榜 +榧 +榨 +榫 +榭 +榰 +榱 +榴 +榷 +榻 +槁 +槃 +槊 +槌 +槎 +槐 +槔 +槚 +槛 +槜 +槟 +槠 +槭 +槱 +槲 +槽 +槿 +樊 +樗 +樘 +樟 +模 +樨 +横 +樯 +樱 +樵 +樽 +樾 +橄 +橇 +橐 +橑 +橘 +橙 +橛 +橞 +橡 +橥 +橦 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檑 +檗 +檞 +檠 +檩 +檫 +檬 +櫆 +欂 +欠 +次 +欢 +欣 +欤 +欧 +欲 +欸 +欹 +欺 +欻 +款 +歃 +歅 +歆 +歇 +歉 +歌 +歙 +止 +正 +此 +步 +武 +歧 +歪 +歹 +死 +歼 +殁 +殂 +殃 +殄 +殆 +殇 +殉 +殊 +残 +殍 +殒 +殓 +殖 +殚 +殛 +殡 +殣 +殪 +殳 +殴 +段 +殷 +殿 +毁 +毂 +毅 +毋 +毌 +母 +每 +毐 +毒 +毓 +比 +毕 +毖 +毗 +毙 +毛 +毡 +毪 +毫 +毯 +毳 +毵 +毹 +毽 +氅 +氆 +氇 +氍 +氏 +氐 +民 +氓 +气 +氕 +氖 +氘 +氙 +氚 +氛 +氟 +氡 +氢 +氤 +氦 +氧 +氨 +氩 +氪 +氮 +氯 +氰 +氲 +水 +永 +氾 +氿 +汀 +汁 +求 +汆 +汇 +汈 +汉 +汊 +汋 +汐 +汔 +汕 +汗 +汛 +汜 +汝 +汞 +江 +池 +污 +汤 +汧 +汨 +汩 +汪 +汫 +汭 +汰 +汲 +汴 +汶 +汹 +汽 +汾 +沁 +沂 +沃 +沄 +沅 +沆 +沇 +沈 +沉 +沌 +沏 +沐 +沓 +沔 +沘 +沙 +沚 +沛 +沟 +没 +沣 +沤 +沥 +沦 +沧 +沨 +沩 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +沺 +治 +沼 +沽 +沾 +沿 +泂 +泃 +泄 +泅 +泇 +泉 +泊 +泌 +泐 +泓 +泔 +法 +泖 +泗 +泙 +泚 +泛 +泜 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泺 +泻 +泼 +泽 +泾 +洁 +洄 +洇 +洈 +洋 +洌 +洎 +洑 +洒 +洓 +洗 +洘 +洙 +洚 +洛 +洞 +洢 +洣 +津 +洧 +洨 +洪 +洫 +洭 +洮 +洱 +洲 +洳 +洴 +洵 +洸 +洹 +洺 +活 +洼 +洽 +派 +洿 +流 +浃 +浅 +浆 +浇 +浈 +浉 +浊 +测 +浍 +济 +浏 +浐 +浑 +浒 +浓 +浔 +浕 +浙 +浚 +浛 +浜 +浞 +浟 +浠 +浡 +浣 +浥 +浦 +浩 +浪 +浬 +浭 +浮 +浯 +浰 +浲 +浴 +海 +浸 +浼 +涂 +涄 +涅 +消 +涉 +涌 +涍 +涎 +涐 +涑 +涓 +涔 +涕 +涘 +涛 +涝 +涞 +涟 +涠 +涡 +涢 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涫 +涮 +涯 +液 +涴 +涵 +涸 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淏 +淑 +淖 +淘 +淙 +淜 +淝 +淞 +淟 +淠 +淡 +淤 +淦 +淫 +淬 +淮 +淯 +深 +淳 +淴 +混 +淹 +添 +淼 +清 +渊 +渌 +渍 +渎 +渐 +渑 +渔 +渗 +渚 +渝 +渟 +渠 +渡 +渣 +渤 +渥 +温 +渫 +渭 +港 +渰 +渲 +渴 +游 +渺 +渼 +湃 +湄 +湉 +湍 +湎 +湑 +湓 +湔 +湖 +湘 +湛 +湜 +湝 +湟 +湣 +湫 +湮 +湲 +湴 +湾 +湿 +溁 +溃 +溅 +溆 +溇 +溉 +溍 +溏 +源 +溘 +溚 +溜 +溞 +溟 +溠 +溢 +溥 +溦 +溧 +溪 +溯 +溱 +溲 +溴 +溵 +溶 +溷 +溹 +溺 +溻 +溽 +滁 +滂 +滃 +滆 +滇 +滉 +滋 +滍 +滏 +滑 +滓 +滔 +滕 +滗 +滘 +滚 +滞 +滟 +滠 +满 +滢 +滤 +滥 +滦 +滧 +滨 +滩 +滪 +滫 +滴 +滹 +漂 +漆 +漈 +漉 +漋 +漏 +漓 +演 +漕 +漖 +漠 +漤 +漦 +漩 +漪 +漫 +漭 +漯 +漱 +漳 +漴 +漶 +漷 +漹 +漻 +漼 +漾 +潆 +潇 +潋 +潍 +潏 +潖 +潘 +潜 +潞 +潟 +潢 +潦 +潩 +潭 +潮 +潲 +潴 +潵 +潸 +潺 +潼 +潽 +潾 +澂 +澄 +澈 +澉 +澌 +澍 +澎 +澛 +澜 +澡 +澥 +澧 +澪 +澭 +澳 +澴 +澶 +澹 +澼 +澽 +激 +濂 +濉 +濋 +濑 +濒 +濞 +濠 +濡 +濩 +濮 +濯 +瀌 +瀍 +瀑 +瀔 +瀚 +瀛 +瀣 +瀱 +瀵 +瀹 +瀼 +灈 +灌 +灏 +灞 +火 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +灾 +灿 +炀 +炅 +炆 +炉 +炊 +炌 +炎 +炒 +炔 +炕 +炖 +炘 +炙 +炜 +炝 +炟 +炣 +炫 +炬 +炭 +炮 +炯 +炱 +炳 +炷 +炸 +点 +炻 +炼 +炽 +烀 +烁 +烂 +烃 +烈 +烊 +烔 +烘 +烙 +烛 +烜 +烝 +烟 +烠 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烶 +烷 +烹 +烺 +烻 +烽 +焆 +焉 +焊 +焌 +焐 +焓 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +焞 +焦 +焯 +焰 +焱 +然 +煁 +煃 +煅 +煊 +煋 +煌 +煎 +煓 +煜 +煞 +煟 +煤 +煦 +照 +煨 +煮 +煲 +煳 +煴 +煸 +煺 +煽 +熄 +熇 +熊 +熏 +熔 +熘 +熙 +熛 +熜 +熟 +熠 +熥 +熨 +熬 +熵 +熹 +熻 +燃 +燊 +燋 +燎 +燏 +燔 +燕 +燚 +燠 +燥 +燧 +燮 +燹 +爆 +爇 +爔 +爚 +爝 +爟 +爨 +爪 +爬 +爰 +爱 +爵 +父 +爷 +爸 +爹 +爻 +爽 +爿 +牁 +牂 +片 +版 +牌 +牍 +牒 +牖 +牙 +牚 +牛 +牝 +牟 +牡 +牢 +牤 +牥 +牦 +牧 +物 +牮 +牯 +牲 +牵 +特 +牺 +牻 +牾 +牿 +犀 +犁 +犄 +犇 +犊 +犋 +犍 +犏 +犒 +犟 +犨 +犬 +犯 +犰 +犴 +状 +犷 +犸 +犹 +狁 +狂 +狃 +狄 +狈 +狉 +狍 +狎 +狐 +狒 +狗 +狙 +狝 +狞 +狠 +狡 +狨 +狩 +独 +狭 +狮 +狯 +狰 +狱 +狲 +狳 +狴 +狷 +狸 +狺 +狻 +狼 +猁 +猃 +猄 +猇 +猊 +猎 +猕 +猖 +猗 +猛 +猜 +猝 +猞 +猡 +猢 +猥 +猩 +猪 +猫 +猬 +献 +猯 +猰 +猱 +猴 +猷 +猹 +猺 +猾 +猿 +獍 +獐 +獒 +獗 +獠 +獬 +獭 +獯 +獴 +獾 +玃 +玄 +率 +玉 +王 +玎 +玑 +玒 +玓 +玕 +玖 +玘 +玙 +玚 +玛 +玞 +玟 +玠 +玡 +玢 +玤 +玥 +玦 +玩 +玫 +玭 +玮 +环 +现 +玱 +玲 +玳 +玶 +玷 +玹 +玺 +玻 +玼 +玿 +珀 +珂 +珅 +珇 +珈 +珉 +珊 +珋 +珌 +珍 +珏 +珐 +珑 +珒 +珕 +珖 +珙 +珛 +珝 +珞 +珠 +珢 +珣 +珥 +珦 +珧 +珩 +珪 +珫 +班 +珰 +珲 +珵 +珷 +珸 +珹 +珺 +珽 +琀 +球 +琄 +琅 +理 +琇 +琈 +琉 +琊 +琎 +琏 +琐 +琔 +琚 +琛 +琟 +琡 +琢 +琤 +琥 +琦 +琨 +琪 +琫 +琬 +琭 +琮 +琯 +琰 +琲 +琳 +琴 +琵 +琶 +琼 +瑀 +瑁 +瑂 +瑃 +瑄 +瑅 +瑆 +瑑 +瑓 +瑔 +瑕 +瑖 +瑗 +瑙 +瑚 +瑛 +瑜 +瑝 +瑞 +瑟 +瑢 +瑧 +瑨 +瑬 +瑭 +瑰 +瑱 +瑳 +瑶 +瑷 +瑾 +璀 +璁 +璃 +璆 +璇 +璈 +璋 +璎 +璐 +璒 +璘 +璜 +璞 +璟 +璠 +璥 +璧 +璨 +璩 +璪 +璬 +璮 +璱 +璲 +璺 +瓀 +瓒 +瓖 +瓘 +瓜 +瓞 +瓠 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +瓻 +瓿 +甄 +甍 +甏 +甑 +甓 +甗 +甘 +甚 +甜 +生 +甡 +甥 +甦 +用 +甩 +甪 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +畈 +畋 +界 +畎 +畏 +畔 +畖 +留 +畚 +畛 +畜 +畤 +略 +畦 +番 +畬 +畯 +畲 +畴 +畸 +畹 +畿 +疁 +疃 +疆 +疍 +疏 +疐 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疠 +疡 +疢 +疣 +疤 +疥 +疫 +疬 +疭 +疮 +疯 +疰 +疱 +疲 +疳 +疴 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +痃 +痄 +病 +症 +痈 +痉 +痊 +痍 +痒 +痓 +痔 +痕 +痘 +痛 +痞 +痢 +痣 +痤 +痦 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痼 +痿 +瘀 +瘁 +瘃 +瘅 +瘆 +瘊 +瘌 +瘐 +瘕 +瘗 +瘘 +瘙 +瘛 +瘟 +瘠 +瘢 +瘤 +瘥 +瘦 +瘩 +瘪 +瘫 +瘭 +瘰 +瘳 +瘴 +瘵 +瘸 +瘼 +瘾 +瘿 +癀 +癃 +癌 +癍 +癔 +癖 +癗 +癜 +癞 +癣 +癫 +癯 +癸 +登 +白 +百 +癿 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皕 +皖 +皙 +皛 +皞 +皤 +皦 +皭 +皮 +皱 +皲 +皴 +皿 +盂 +盅 +盆 +盈 +盉 +益 +盍 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盟 +盥 +盦 +目 +盯 +盱 +盲 +直 +盷 +相 +盹 +盼 +盾 +省 +眄 +眇 +眈 +眉 +眊 +看 +眍 +眙 +眚 +真 +眠 +眢 +眦 +眨 +眩 +眬 +眭 +眯 +眵 +眶 +眷 +眸 +眺 +眼 +着 +睁 +睃 +睄 +睇 +睎 +睐 +睑 +睚 +睛 +睡 +睢 +督 +睥 +睦 +睨 +睫 +睬 +睹 +睽 +睾 +睿 +瞀 +瞄 +瞅 +瞋 +瞌 +瞍 +瞎 +瞑 +瞒 +瞟 +瞠 +瞢 +瞥 +瞧 +瞩 +瞪 +瞫 +瞬 +瞭 +瞰 +瞳 +瞵 +瞻 +瞽 +瞿 +矍 +矗 +矛 +矜 +矞 +矢 +矣 +知 +矧 +矩 +矫 +矬 +短 +矮 +矰 +石 +矶 +矸 +矻 +矼 +矾 +矿 +砀 +码 +砂 +砄 +砆 +砉 +砌 +砍 +砑 +砒 +研 +砖 +砗 +砘 +砚 +砜 +砝 +砟 +砠 +砣 +砥 +砧 +砫 +砬 +砭 +砮 +砰 +破 +砵 +砷 +砸 +砹 +砺 +砻 +砼 +砾 +础 +硁 +硅 +硇 +硊 +硌 +硍 +硎 +硐 +硒 +硔 +硕 +硖 +硗 +硙 +硚 +硝 +硪 +硫 +硬 +硭 +确 +硼 +硿 +碃 +碇 +碈 +碉 +碌 +碍 +碎 +碏 +碑 +碓 +碗 +碘 +碚 +碛 +碜 +碟 +碡 +碣 +碥 +碧 +碨 +碰 +碱 +碲 +碳 +碴 +碶 +碹 +碾 +磁 +磅 +磉 +磊 +磋 +磏 +磐 +磔 +磕 +磙 +磜 +磡 +磨 +磬 +磲 +磴 +磷 +磹 +磻 +礁 +礅 +礌 +礓 +礞 +礴 +礵 +示 +礼 +社 +祀 +祁 +祃 +祆 +祇 +祈 +祉 +祊 +祋 +祎 +祏 +祐 +祓 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +祧 +票 +祭 +祯 +祲 +祷 +祸 +祺 +祼 +祾 +禀 +禁 +禄 +禅 +禊 +禋 +福 +禒 +禔 +禘 +禚 +禛 +禤 +禧 +禳 +禹 +禺 +离 +禽 +禾 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秕 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秫 +秬 +秭 +积 +称 +秸 +移 +秽 +秾 +稀 +稂 +稃 +稆 +程 +稌 +稍 +税 +稑 +稔 +稗 +稙 +稚 +稞 +稠 +稣 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穄 +穆 +穑 +穗 +穙 +穜 +穟 +穰 +穴 +究 +穷 +穸 +穹 +空 +穿 +窀 +突 +窃 +窄 +窅 +窈 +窊 +窍 +窎 +窑 +窒 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窣 +窥 +窦 +窨 +窬 +窭 +窳 +窸 +窿 +立 +竑 +竖 +竘 +站 +竞 +竟 +章 +竣 +童 +竦 +竫 +竭 +端 +竹 +竺 +竽 +竿 +笃 +笄 +笆 +笈 +笊 +笋 +笏 +笑 +笔 +笕 +笙 +笛 +笞 +笠 +笤 +笥 +符 +笨 +笪 +笫 +第 +笮 +笯 +笱 +笳 +笸 +笺 +笼 +笾 +筀 +筅 +筇 +等 +筋 +筌 +筏 +筐 +筑 +筒 +答 +策 +筘 +筚 +筛 +筜 +筝 +筠 +筢 +筤 +筥 +筦 +筮 +筱 +筲 +筵 +筶 +筷 +筹 +筻 +筼 +签 +简 +箅 +箍 +箐 +箓 +箔 +箕 +箖 +算 +箜 +管 +箢 +箦 +箧 +箨 +箩 +箪 +箫 +箬 +箭 +箱 +箴 +箸 +篁 +篆 +篇 +篌 +篑 +篓 +篙 +篚 +篝 +篡 +篥 +篦 +篪 +篮 +篯 +篱 +篷 +篼 +篾 +簃 +簇 +簉 +簋 +簌 +簏 +簕 +簖 +簝 +簟 +簠 +簧 +簪 +簰 +簸 +簿 +籀 +籁 +籍 +籥 +米 +籴 +类 +籼 +籽 +粉 +粑 +粒 +粕 +粗 +粘 +粜 +粝 +粞 +粟 +粢 +粤 +粥 +粪 +粮 +粱 +粲 +粳 +粹 +粼 +粽 +精 +粿 +糁 +糅 +糇 +糈 +糊 +糌 +糍 +糒 +糕 +糖 +糗 +糙 +糜 +糟 +糠 +糨 +糯 +糵 +系 +紊 +素 +索 +紧 +紫 +累 +絜 +絮 +絷 +綦 +綮 +縠 +縢 +縻 +繁 +繄 +繇 +纂 +纛 +纠 +纡 +红 +纣 +纤 +纥 +约 +级 +纨 +纩 +纪 +纫 +纬 +纭 +纮 +纯 +纰 +纱 +纲 +纳 +纴 +纵 +纶 +纷 +纸 +纹 +纺 +纻 +纼 +纽 +纾 +线 +绀 +绁 +绂 +练 +组 +绅 +细 +织 +终 +绉 +绊 +绋 +绌 +绍 +绎 +经 +绐 +绑 +绒 +结 +绔 +绕 +绖 +绗 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绠 +绡 +绢 +绣 +绤 +绥 +绦 +继 +绨 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绱 +绲 +绳 +维 +绵 +绶 +绷 +绸 +绹 +绺 +绻 +综 +绽 +绾 +绿 +缀 +缁 +缂 +缃 +缄 +缅 +缆 +缇 +缈 +缉 +缊 +缌 +缎 +缐 +缑 +缒 +缓 +缔 +缕 +编 +缗 +缘 +缙 +缚 +缛 +缜 +缝 +缞 +缟 +缠 +缡 +缢 +缣 +缤 +缥 +缦 +缧 +缨 +缩 +缪 +缫 +缬 +缭 +缮 +缯 +缰 +缱 +缲 +缳 +缴 +缵 +缶 +缸 +缺 +罂 +罄 +罅 +罍 +罐 +网 +罔 +罕 +罗 +罘 +罚 +罟 +罡 +罢 +罨 +罩 +罪 +置 +罱 +署 +罴 +罶 +罹 +罽 +罾 +羁 +羊 +羌 +美 +羑 +羓 +羔 +羕 +羖 +羚 +羝 +羞 +羟 +羡 +群 +羧 +羯 +羰 +羱 +羲 +羸 +羹 +羼 +羽 +羿 +翀 +翁 +翂 +翃 +翅 +翈 +翊 +翌 +翎 +翔 +翕 +翘 +翙 +翚 +翛 +翟 +翠 +翡 +翥 +翦 +翩 +翮 +翯 +翰 +翱 +翳 +翷 +翻 +翼 +翾 +耀 +老 +考 +耄 +者 +耆 +耇 +耋 +而 +耍 +耏 +耐 +耑 +耒 +耔 +耕 +耖 +耗 +耘 +耙 +耜 +耠 +耢 +耤 +耥 +耦 +耧 +耨 +耩 +耪 +耰 +耱 +耳 +耵 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聃 +聆 +聊 +聋 +职 +聍 +聒 +联 +聘 +聚 +聩 +聪 +聱 +聿 +肃 +肄 +肆 +肇 +肉 +肋 +肌 +肓 +肖 +肘 +肚 +肛 +肝 +肟 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肫 +肭 +肮 +肯 +肱 +育 +肴 +肷 +肸 +肺 +肼 +肽 +肾 +肿 +胀 +胁 +胂 +胃 +胄 +胆 +胈 +背 +胍 +胎 +胖 +胗 +胙 +胚 +胛 +胜 +胝 +胞 +胠 +胡 +胣 +胤 +胥 +胧 +胨 +胩 +胪 +胫 +胬 +胭 +胯 +胰 +胱 +胲 +胳 +胴 +胶 +胸 +胺 +胼 +能 +脂 +脆 +脉 +脊 +脍 +脎 +脏 +脐 +脑 +脒 +脓 +脔 +脖 +脘 +脚 +脞 +脟 +脩 +脬 +脯 +脱 +脲 +脶 +脸 +脾 +脿 +腆 +腈 +腊 +腋 +腌 +腐 +腑 +腒 +腓 +腔 +腕 +腘 +腙 +腚 +腠 +腥 +腧 +腨 +腩 +腭 +腮 +腯 +腰 +腱 +腴 +腹 +腺 +腻 +腼 +腽 +腾 +腿 +膀 +膂 +膈 +膊 +膏 +膑 +膘 +膙 +膛 +膜 +膝 +膦 +膨 +膳 +膺 +膻 +臀 +臂 +臃 +臆 +臊 +臌 +臑 +臜 +臣 +臧 +自 +臬 +臭 +至 +致 +臻 +臼 +臾 +舀 +舁 +舂 +舄 +舅 +舆 +舌 +舍 +舐 +舒 +舔 +舛 +舜 +舞 +舟 +舠 +舢 +舣 +舥 +航 +舫 +般 +舭 +舯 +舰 +舱 +舲 +舳 +舴 +舵 +舶 +舷 +舸 +船 +舻 +舾 +艄 +艅 +艇 +艉 +艋 +艎 +艏 +艘 +艚 +艟 +艨 +艮 +良 +艰 +色 +艳 +艴 +艺 +艽 +艾 +艿 +节 +芃 +芄 +芈 +芊 +芋 +芍 +芎 +芏 +芑 +芒 +芗 +芘 +芙 +芜 +芝 +芟 +芠 +芡 +芣 +芤 +芥 +芦 +芨 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +芰 +花 +芳 +芴 +芷 +芸 +芹 +芼 +芽 +芾 +苁 +苄 +苇 +苈 +苉 +苊 +苋 +苌 +苍 +苎 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苘 +苛 +苜 +苞 +苟 +苠 +苡 +苣 +苤 +若 +苦 +苧 +苫 +苯 +英 +苴 +苷 +苹 +苻 +苾 +茀 +茁 +茂 +范 +茄 +茅 +茆 +茈 +茉 +茋 +茌 +茎 +茏 +茑 +茓 +茔 +茕 +茗 +茚 +茛 +茜 +茝 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茳 +茴 +茵 +茶 +茸 +茹 +茺 +茼 +茽 +荀 +荁 +荃 +荄 +荆 +荇 +草 +荏 +荐 +荑 +荒 +荓 +荔 +荖 +荙 +荚 +荛 +荜 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荦 +荧 +荨 +荩 +荪 +荫 +荬 +荭 +荮 +药 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莎 +莒 +莓 +莘 +莙 +莛 +莜 +莝 +莞 +莠 +莨 +莩 +莪 +莫 +莰 +莱 +莲 +莳 +莴 +莶 +获 +莸 +莹 +莺 +莼 +莽 +莿 +菀 +菁 +菂 +菅 +菇 +菉 +菊 +菌 +菍 +菏 +菔 +菖 +菘 +菜 +菝 +菟 +菠 +菡 +菥 +菩 +菪 +菰 +菱 +菲 +菹 +菼 +菽 +萁 +萃 +萄 +萆 +萋 +萌 +萍 +萎 +萏 +萑 +萘 +萚 +萜 +萝 +萣 +萤 +营 +萦 +萧 +萨 +萩 +萱 +萳 +萸 +萹 +萼 +落 +葆 +葎 +葑 +葖 +著 +葙 +葚 +葛 +葜 +葡 +董 +葩 +葫 +葬 +葭 +葰 +葱 +葳 +葴 +葵 +葶 +葸 +葺 +蒂 +蒄 +蒇 +蒈 +蒉 +蒋 +蒌 +蒎 +蒐 +蒗 +蒙 +蒜 +蒟 +蒡 +蒨 +蒯 +蒱 +蒲 +蒴 +蒸 +蒹 +蒺 +蒻 +蒽 +蒿 +蓁 +蓂 +蓄 +蓇 +蓉 +蓊 +蓍 +蓏 +蓐 +蓑 +蓓 +蓖 +蓝 +蓟 +蓠 +蓢 +蓣 +蓥 +蓦 +蓬 +蓰 +蓼 +蓿 +蔀 +蔃 +蔈 +蔊 +蔌 +蔑 +蔓 +蔗 +蔚 +蔟 +蔡 +蔫 +蔬 +蔷 +蔸 +蔹 +蔺 +蔻 +蔼 +蔽 +蕃 +蕈 +蕉 +蕊 +蕖 +蕗 +蕙 +蕞 +蕤 +蕨 +蕰 +蕲 +蕴 +蕹 +蕺 +蕻 +蕾 +薁 +薄 +薅 +薇 +薏 +薛 +薜 +薢 +薤 +薨 +薪 +薮 +薯 +薰 +薳 +薷 +薸 +薹 +薿 +藁 +藉 +藏 +藐 +藓 +藕 +藜 +藟 +藠 +藤 +藦 +藨 +藩 +藻 +藿 +蘅 +蘑 +蘖 +蘘 +蘧 +蘩 +蘸 +蘼 +虎 +虏 +虐 +虑 +虒 +虓 +虔 +虚 +虞 +虢 +虤 +虫 +虬 +虮 +虱 +虷 +虸 +虹 +虺 +虻 +虼 +虽 +虾 +虿 +蚀 +蚁 +蚂 +蚄 +蚆 +蚊 +蚋 +蚌 +蚍 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚧 +蚨 +蚩 +蚪 +蚬 +蚯 +蚰 +蚱 +蚲 +蚴 +蚶 +蚺 +蛀 +蛃 +蛄 +蛆 +蛇 +蛉 +蛊 +蛋 +蛎 +蛏 +蛐 +蛑 +蛔 +蛘 +蛙 +蛛 +蛞 +蛟 +蛤 +蛩 +蛭 +蛮 +蛰 +蛱 +蛲 +蛳 +蛴 +蛸 +蛹 +蛾 +蜀 +蜂 +蜃 +蜇 +蜈 +蜉 +蜊 +蜍 +蜎 +蜐 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜞 +蜡 +蜢 +蜣 +蜥 +蜩 +蜮 +蜱 +蜴 +蜷 +蜻 +蜾 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝓 +蝗 +蝘 +蝙 +蝠 +蝣 +蝤 +蝥 +蝮 +蝰 +蝲 +蝴 +蝶 +蝻 +蝼 +蝽 +蝾 +螂 +螃 +螅 +螈 +螋 +融 +螗 +螟 +螠 +螣 +螨 +螫 +螬 +螭 +螯 +螱 +螳 +螵 +螺 +螽 +蟀 +蟆 +蟊 +蟋 +蟏 +蟑 +蟒 +蟛 +蟠 +蟥 +蟪 +蟫 +蟮 +蟹 +蟾 +蠃 +蠊 +蠋 +蠓 +蠕 +蠖 +蠡 +蠢 +蠲 +蠹 +蠼 +血 +衃 +衄 +衅 +行 +衍 +衎 +衒 +衔 +街 +衙 +衠 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衽 +衾 +衿 +袁 +袂 +袄 +袅 +袆 +袈 +袋 +袍 +袒 +袖 +袗 +袜 +袢 +袤 +袪 +被 +袭 +袯 +袱 +袷 +袼 +裁 +裂 +装 +裆 +裈 +裉 +裎 +裒 +裔 +裕 +裘 +裙 +裛 +裟 +裢 +裣 +裤 +裥 +裨 +裰 +裱 +裳 +裴 +裸 +裹 +裼 +裾 +褂 +褊 +褐 +褒 +褓 +褕 +褙 +褚 +褛 +褟 +褡 +褥 +褪 +褫 +褯 +褰 +褴 +褶 +襁 +襄 +襕 +襚 +襜 +襞 +襟 +襦 +襫 +襻 +西 +要 +覃 +覆 +见 +观 +觃 +规 +觅 +视 +觇 +览 +觉 +觊 +觋 +觌 +觎 +觏 +觐 +觑 +角 +觖 +觚 +觜 +觞 +觟 +解 +觥 +触 +觫 +觭 +觯 +觱 +觳 +觿 +言 +訄 +訇 +訚 +訾 +詈 +詟 +詹 +誉 +誊 +誓 +謇 +警 +譬 +计 +订 +讣 +认 +讥 +讦 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讱 +讲 +讳 +讴 +讵 +讶 +讷 +许 +讹 +论 +讻 +讼 +讽 +设 +访 +诀 +证 +诂 +诃 +评 +诅 +识 +诇 +诈 +诉 +诊 +诋 +诌 +词 +诎 +诏 +诐 +译 +诒 +诓 +诔 +试 +诖 +诗 +诘 +诙 +诚 +诛 +诜 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诨 +诩 +诫 +诬 +语 +诮 +误 +诰 +诱 +诲 +诳 +说 +诵 +请 +诸 +诹 +诺 +读 +诼 +诽 +课 +诿 +谀 +谁 +谂 +调 +谄 +谅 +谆 +谇 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谖 +谗 +谙 +谚 +谛 +谜 +谝 +谞 +谟 +谠 +谡 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谫 +谬 +谭 +谮 +谯 +谰 +谱 +谲 +谳 +谴 +谵 +谶 +谷 +谼 +谿 +豁 +豆 +豇 +豉 +豌 +豕 +豚 +象 +豢 +豨 +豪 +豫 +豮 +豳 +豸 +豹 +豺 +貂 +貅 +貆 +貉 +貊 +貌 +貔 +貘 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贳 +贴 +贵 +贶 +贷 +贸 +费 +贺 +贻 +贼 +贽 +贾 +贿 +赀 +赁 +赂 +赃 +资 +赅 +赆 +赇 +赈 +赉 +赊 +赋 +赌 +赍 +赎 +赏 +赐 +赑 +赒 +赓 +赔 +赕 +赖 +赗 +赘 +赙 +赚 +赛 +赜 +赝 +赞 +赟 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赪 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +趄 +超 +越 +趋 +趑 +趔 +趟 +趣 +趯 +趱 +足 +趴 +趵 +趸 +趺 +趼 +趾 +趿 +跂 +跃 +跄 +跆 +跋 +跌 +跎 +跏 +跐 +跑 +跖 +跗 +跚 +跛 +距 +跞 +跟 +跣 +跤 +跨 +跪 +跬 +路 +跱 +跳 +践 +跶 +跷 +跸 +跹 +跺 +跻 +跽 +踅 +踉 +踊 +踌 +踏 +踒 +踔 +踝 +踞 +踟 +踢 +踣 +踦 +踩 +踪 +踬 +踮 +踯 +踱 +踵 +踶 +踹 +踺 +踽 +蹀 +蹁 +蹂 +蹄 +蹅 +蹇 +蹈 +蹉 +蹊 +蹋 +蹐 +蹑 +蹒 +蹙 +蹚 +蹜 +蹢 +蹦 +蹩 +蹬 +蹭 +蹯 +蹰 +蹲 +蹴 +蹶 +蹼 +蹽 +蹾 +蹿 +躁 +躅 +躇 +躏 +躐 +躔 +躜 +躞 +身 +躬 +躯 +躲 +躺 +车 +轧 +轨 +轩 +轪 +轫 +转 +轭 +轮 +软 +轰 +轱 +轲 +轳 +轴 +轵 +轶 +轷 +轸 +轹 +轺 +轻 +轼 +载 +轾 +轿 +辀 +辁 +辂 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辋 +辌 +辍 +辎 +辏 +辐 +辑 +辒 +输 +辔 +辕 +辖 +辗 +辘 +辙 +辚 +辛 +辜 +辞 +辟 +辣 +辨 +辩 +辫 +辰 +辱 +边 +辽 +达 +辿 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +迓 +返 +迕 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迨 +迩 +迪 +迫 +迭 +迮 +述 +迳 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逄 +逅 +逆 +选 +逊 +逋 +逍 +透 +逐 +逑 +递 +途 +逖 +逗 +通 +逛 +逝 +逞 +速 +造 +逡 +逢 +逦 +逭 +逮 +逯 +逴 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遄 +遆 +遇 +遍 +遏 +遐 +遑 +遒 +道 +遗 +遘 +遛 +遢 +遣 +遥 +遨 +遭 +遮 +遴 +遵 +遹 +遽 +避 +邀 +邂 +邃 +邈 +邋 +邑 +邓 +邕 +邗 +邘 +邙 +邛 +邝 +邠 +邡 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邲 +邳 +邴 +邵 +邶 +邸 +邹 +邺 +邻 +邽 +邾 +邿 +郁 +郃 +郄 +郅 +郇 +郈 +郊 +郎 +郏 +郐 +郑 +郓 +郗 +郚 +郛 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郪 +郫 +郭 +郯 +郴 +郸 +都 +郾 +郿 +鄀 +鄂 +鄃 +鄄 +鄅 +鄌 +鄑 +鄗 +鄘 +鄙 +鄚 +鄜 +鄞 +鄠 +鄢 +鄣 +鄫 +鄯 +鄱 +鄹 +酂 +酃 +酅 +酆 +酉 +酊 +酋 +酌 +配 +酎 +酏 +酐 +酒 +酗 +酚 +酝 +酞 +酡 +酢 +酣 +酤 +酥 +酦 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酲 +酴 +酵 +酶 +酷 +酸 +酹 +酺 +酽 +酾 +酿 +醅 +醇 +醉 +醋 +醌 +醍 +醐 +醑 +醒 +醚 +醛 +醢 +醨 +醪 +醭 +醮 +醯 +醴 +醵 +醺 +醾 +采 +釉 +释 +里 +重 +野 +量 +釐 +金 +釜 +鉴 +銎 +銮 +鋆 +鋈 +錾 +鍪 +鎏 +鏊 +鏖 +鐾 +鑫 +钆 +钇 +针 +钉 +钊 +钋 +钌 +钍 +钎 +钏 +钐 +钒 +钓 +钔 +钕 +钖 +钗 +钘 +钙 +钚 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钪 +钫 +钬 +钭 +钮 +钯 +钰 +钱 +钲 +钳 +钴 +钵 +钷 +钹 +钺 +钻 +钼 +钽 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铈 +铉 +铊 +铋 +铌 +铍 +铎 +铏 +铐 +铑 +铒 +铕 +铖 +铗 +铘 +铙 +铚 +铛 +铜 +铝 +铞 +铟 +铠 +铡 +铢 +铣 +铤 +铥 +铧 +铨 +铩 +铪 +铫 +铬 +铭 +铮 +铯 +铰 +铱 +铲 +铳 +铴 +铵 +银 +铷 +铸 +铹 +铺 +铻 +铼 +铽 +链 +铿 +销 +锁 +锂 +锃 +锄 +锅 +锆 +锇 +锈 +锉 +锊 +锋 +锌 +锍 +锎 +锏 +锐 +锑 +锒 +锓 +锔 +锕 +锖 +锗 +锘 +错 +锚 +锛 +锜 +锝 +锞 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锧 +锨 +锩 +锪 +锫 +锬 +锭 +键 +锯 +锰 +锱 +锲 +锳 +锴 +锵 +锶 +锷 +锸 +锹 +锺 +锻 +锼 +锽 +锾 +锿 +镀 +镁 +镂 +镃 +镄 +镅 +镆 +镇 +镈 +镉 +镊 +镋 +镌 +镍 +镎 +镏 +镐 +镑 +镒 +镓 +镔 +镕 +镖 +镗 +镘 +镚 +镛 +镜 +镝 +镞 +镠 +镡 +镢 +镣 +镤 +镥 +镦 +镧 +镨 +镩 +镪 +镫 +镬 +镭 +镮 +镯 +镰 +镱 +镲 +镳 +镴 +镵 +镶 +长 +门 +闩 +闪 +闫 +闭 +问 +闯 +闰 +闱 +闲 +闳 +间 +闵 +闶 +闷 +闸 +闹 +闺 +闻 +闼 +闽 +闾 +闿 +阀 +阁 +阂 +阃 +阄 +阅 +阆 +阇 +阈 +阉 +阊 +阋 +阌 +阍 +阎 +阏 +阐 +阑 +阒 +阔 +阕 +阖 +阗 +阘 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阼 +阽 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陉 +陋 +陌 +降 +陎 +限 +陑 +陔 +陕 +陛 +陞 +陟 +陡 +院 +除 +陧 +陨 +险 +陪 +陬 +陲 +陴 +陵 +陶 +陷 +隃 +隅 +隆 +隈 +隋 +隍 +随 +隐 +隔 +隗 +隘 +隙 +障 +隧 +隩 +隰 +隳 +隶 +隹 +隺 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雊 +雌 +雍 +雎 +雏 +雒 +雕 +雠 +雨 +雩 +雪 +雯 +雱 +雳 +零 +雷 +雹 +雾 +需 +霁 +霄 +霅 +霆 +震 +霈 +霉 +霍 +霎 +霏 +霓 +霖 +霜 +霞 +霨 +霪 +霭 +霰 +露 +霸 +霹 +霾 +青 +靓 +靖 +静 +靛 +非 +靠 +靡 +面 +靥 +革 +靬 +靰 +靳 +靴 +靶 +靸 +靺 +靼 +靽 +靿 +鞁 +鞅 +鞋 +鞍 +鞑 +鞒 +鞔 +鞘 +鞠 +鞡 +鞣 +鞧 +鞨 +鞫 +鞬 +鞭 +鞮 +鞯 +鞲 +鞳 +鞴 +韂 +韦 +韧 +韨 +韩 +韪 +韫 +韬 +韭 +音 +韵 +韶 +页 +顶 +顷 +顸 +项 +顺 +须 +顼 +顽 +顾 +顿 +颀 +颁 +颂 +颃 +预 +颅 +领 +颇 +颈 +颉 +颊 +颋 +颌 +颍 +颎 +颏 +颐 +频 +颓 +颔 +颖 +颗 +题 +颙 +颚 +颛 +颜 +额 +颞 +颟 +颠 +颡 +颢 +颤 +颥 +颦 +颧 +风 +飏 +飐 +飑 +飒 +飓 +飔 +飕 +飗 +飘 +飙 +飞 +食 +飧 +飨 +餍 +餐 +餮 +饔 +饕 +饥 +饧 +饨 +饩 +饪 +饫 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饳 +饴 +饵 +饶 +饷 +饸 +饹 +饺 +饻 +饼 +饽 +饿 +馁 +馃 +馄 +馅 +馆 +馇 +馈 +馉 +馊 +馋 +馌 +馍 +馏 +馐 +馑 +馒 +馓 +馔 +馕 +首 +馗 +馘 +香 +馝 +馞 +馥 +馧 +馨 +马 +驭 +驮 +驯 +驰 +驱 +驲 +驳 +驴 +驵 +驶 +驷 +驸 +驹 +驺 +驻 +驼 +驽 +驾 +驿 +骀 +骁 +骂 +骃 +骄 +骅 +骆 +骇 +骈 +骉 +骊 +骋 +验 +骍 +骎 +骏 +骐 +骑 +骒 +骓 +骕 +骖 +骗 +骘 +骙 +骚 +骛 +骜 +骝 +骞 +骟 +骠 +骡 +骢 +骣 +骤 +骥 +骦 +骧 +骨 +骰 +骱 +骶 +骷 +骸 +骺 +骼 +髀 +髁 +髂 +髃 +髅 +髋 +髌 +髎 +髑 +髓 +高 +髡 +髢 +髦 +髫 +髭 +髯 +髹 +髻 +髽 +鬃 +鬈 +鬏 +鬒 +鬓 +鬘 +鬟 +鬣 +鬯 +鬲 +鬶 +鬷 +鬻 +鬼 +魁 +魂 +魃 +魄 +魅 +魆 +魇 +魈 +魉 +魋 +魍 +魏 +魑 +魔 +鱼 +鱽 +鱾 +鱿 +鲀 +鲁 +鲂 +鲃 +鲅 +鲆 +鲇 +鲈 +鲉 +鲊 +鲋 +鲌 +鲍 +鲎 +鲏 +鲐 +鲑 +鲒 +鲔 +鲕 +鲖 +鲗 +鲘 +鲙 +鲚 +鲛 +鲜 +鲝 +鲞 +鲟 +鲠 +鲡 +鲢 +鲣 +鲤 +鲥 +鲦 +鲧 +鲨 +鲩 +鲪 +鲫 +鲬 +鲭 +鲮 +鲯 +鲰 +鲱 +鲲 +鲳 +鲴 +鲵 +鲷 +鲸 +鲹 +鲺 +鲻 +鲼 +鲽 +鲾 +鲿 +鳀 +鳁 +鳂 +鳃 +鳄 +鳅 +鳇 +鳈 +鳉 +鳊 +鳌 +鳍 +鳎 +鳏 +鳐 +鳑 +鳒 +鳓 +鳔 +鳕 +鳖 +鳗 +鳘 +鳙 +鳚 +鳛 +鳜 +鳝 +鳞 +鳟 +鳠 +鳡 +鳢 +鳣 +鳤 +鸟 +鸠 +鸡 +鸢 +鸣 +鸤 +鸥 +鸦 +鸧 +鸨 +鸩 +鸪 +鸫 +鸬 +鸭 +鸮 +鸯 +鸰 +鸱 +鸲 +鸳 +鸵 +鸶 +鸷 +鸸 +鸹 +鸺 +鸻 +鸼 +鸽 +鸾 +鸿 +鹀 +鹁 +鹂 +鹃 +鹄 +鹅 +鹆 +鹇 +鹈 +鹉 +鹊 +鹋 +鹌 +鹍 +鹎 +鹏 +鹐 +鹑 +鹒 +鹔 +鹕 +鹖 +鹗 +鹘 +鹙 +鹚 +鹛 +鹜 +鹝 +鹞 +鹟 +鹠 +鹡 +鹢 +鹣 +鹤 +鹦 +鹧 +鹨 +鹩 +鹪 +鹫 +鹬 +鹭 +鹮 +鹯 +鹰 +鹱 +鹲 +鹳 +鹴 +鹾 +鹿 +麀 +麂 +麇 +麈 +麋 +麑 +麒 +麓 +麖 +麝 +麟 +麦 +麸 +麹 +麻 +麽 +麾 +黄 +黇 +黉 +黍 +黎 +黏 +黑 +黔 +默 +黛 +黜 +黝 +黟 +黠 +黡 +黢 +黥 +黧 +黩 +黪 +黯 +黹 +黻 +黼 +黾 +鼋 +鼍 +鼎 +鼐 +鼒 +鼓 +鼗 +鼙 +鼠 +鼢 +鼩 +鼫 +鼬 +鼯 +鼱 +鼷 +鼹 +鼻 +鼽 +鼾 +齁 +齇 +齉 +齐 +齑 +齿 +龀 +龁 +龂 +龃 +龄 +龅 +龆 +龇 +龈 +龉 +龊 +龋 +龌 +龙 +龚 +龛 +龟 +龠 +龢 +鿍 +鿎 +鿏 +㑇 +㑊 +㕮 +㘎 +㙍 +㙘 +㙦 +㛃 +㛚 +㛹 +㟃 +㠇 +㠓 +㤘 +㥄 +㧐 +㧑 +㧟 +㫰 +㬊 +㬎 +㬚 +㭎 +㭕 +㮾 +㰀 +㳇 +㳘 +㳚 +㴔 +㵐 +㶲 +㸆 +㸌 +㺄 +㻬 +㽏 +㿠 +䁖 +䂮 +䃅 +䃎 +䅟 +䌹 +䎃 +䎖 +䏝 +䏡 +䏲 +䐃 +䓖 +䓛 +䓨 +䓫 +䓬 +䗖 +䗛 +䗪 +䗴 +䜣 +䝙 +䢺 +䢼 +䣘 +䥽 +䦃 +䲟 +䲠 +䲢 +䴓 +䴔 +䴕 +䴖 +䴗 +䴘 +䴙 +䶮 +𠅤 +𠙶 +𠳐 +𡎚 +𡐓 +𣗋 +𣲗 +𣲘 +𣸣 +𤧛 +𤩽 +𤫉 +𥔲 +𥕢 +𥖨 +𥻗 +𦈡 +𦒍 +𦙶 +𦝼 +𦭜 +𦰡 +𧿹 +𨐈 +𨙸 +𨚕 +𨟠 +𨭉 +𨱇 +𨱏 +𨱑 +𨱔 +𨺙 +𩽾 +𩾃 +𩾌 +𪟝 +𪣻 +𪤗 +𪨰 +𪨶 +𪩘 +𪾢 +𫄧 +𫄨 +𫄷 +𫄸 +𫇭 +𫌀 +𫍣 +𫍯 +𫍲 +𫍽 +𫐄 +𫐐 +𫐓 +𫑡 +𫓧 +𫓯 +𫓶 +𫓹 +𫔍 +𫔎 +𫔶 +𫖮 +𫖯 +𫖳 +𫗧 +𫗴 +𫘜 +𫘝 +𫘦 +𫘧 +𫘨 +𫘪 +𫘬 +𫚕 +𫚖 +𫚭 +𫛭 +𫞩 +𫟅 +𫟦 +𫟹 +𫟼 +𫠆 +𫠊 +𫠜 +𫢸 +𫫇 +𫭟 +𫭢 +𫭼 +𫮃 +𫰛 +𫵷 +𫶇 +𫷷 +𫸩 +𬀩 +𬀪 +𬂩 +𬃊 +𬇕 +𬇙 +𬇹 +𬉼 +𬊈 +𬊤 +𬌗 +𬍛 +𬍡 +𬍤 +𬒈 +𬒔 +𬒗 +𬕂 +𬘓 +𬘘 +𬘡 +𬘩 +𬘫 +𬘬 +𬘭 +𬘯 +𬙂 +𬙊 +𬙋 +𬜬 +𬜯 +𬞟 +𬟁 +𬟽 +𬣙 +𬣞 +𬣡 +𬣳 +𬤇 +𬤊 +𬤝 +𬨂 +𬨎 +𬩽 +𬪩 +𬬩 +𬬭 +𬬮 +𬬱 +𬬸 +𬬹 +𬬻 +𬬿 +𬭁 +𬭊 +𬭎 +𬭚 +𬭛 +𬭤 +𬭩 +𬭬 +𬭯 +𬭳 +𬭶 +𬭸 +𬭼 +𬮱 +𬮿 +𬯀 +𬯎 +𬱖 +𬱟 +𬳵 +𬳶 +𬳽 +𬳿 +𬴂 +𬴃 +𬴊 +𬶋 +𬶍 +𬶏 +𬶐 +𬶟 +𬶠 +𬶨 +𬶭 +𬶮 +𬷕 +𬸘 +𬸚 +𬸣 +𬸦 +𬸪 +𬹼 +𬺈 +𬺓 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/char/fullwidth_to_halfwidth.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/char/fullwidth_to_halfwidth.tsv new file mode 100644 index 000000000..7b3fd042c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/char/fullwidth_to_halfwidth.tsv @@ -0,0 +1,92 @@ +！ ! +＂ " +＃ # +＄ $ +％ % +＆ & +＇ ' +（ ( +） ) +＊ * +＋ + +， , +－ - +． . +／ / +０ 0 +１ 1 +２ 2 +３ 3 +４ 4 +５ 5 +６ 6 +７ 7 +８ 8 +９ 9 +； ; +＜ < +＝ = +＞ > +？ ? +＠ @ +Ａ A +Ｂ B +Ｃ C +Ｄ D +Ｅ E +Ｆ F +Ｇ G +Ｈ H +Ｉ I +Ｊ J +Ｋ K +Ｌ L +Ｍ M +Ｎ N +Ｏ O +Ｐ P +Ｑ Q +Ｒ R +Ｓ S +Ｔ T +Ｕ U +Ｖ V +Ｗ W +Ｘ X +Ｙ Y +Ｚ Z +＼ \ +＾ ^ +＿ _ +｀ ` +ａ a +ｂ b +ｃ c +ｄ d +ｅ e +ｆ f +ｇ g +ｈ h +ｉ i +ｊ j +ｋ k +ｌ l +ｍ m +ｎ n +ｏ o +ｐ p +ｑ q +ｒ r +ｓ s +ｔ t +ｕ u +ｖ v +ｗ w +ｘ x +ｙ y +ｚ z +｛ { +｜ | +： : +｝ } +～ ~ diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/char/oov_tags.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/char/oov_tags.tsv new file mode 100644 index 000000000..cd6c56f53 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/char/oov_tags.tsv @@ -0,0 +1 @@ + diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/char/punctuations_zh.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/char/punctuations_zh.tsv new file mode 100644 index 000000000..963b07d12 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/char/punctuations_zh.tsv @@ -0,0 +1,72 @@ +！ +？ +｡ +。 +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +／ +： +； +＜ +＝ +＞ +＠ +［ +＼ +］ +＾ +＿ +｀ +｛ +｜ +｝ +～ +｟ +｠ +｢ +｣ +､ +、 +〃 +》 +「 +」 +『 +』 +【 +】 +〔 +〕 +〖 +〗 +〘 +〙 +〚 +〛 +〜 +〝 +〞 +〟 +〰 +– +— +‘ +’ +‛ +“ +” +„ +‟ +… +‧ +﹏ diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/char/upper_to_lower.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/char/upper_to_lower.tsv new file mode 100644 index 000000000..1b603c530 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/char/upper_to_lower.tsv @@ -0,0 +1,9 @@ +壹 一 +贰 二 +叁 三 +肆 四 +伍 五 +陆 六 +柒 七 +捌 八 +玖 九 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/date.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/date.tsv new file mode 100644 index 000000000..ff1f13182 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/date.tsv @@ -0,0 +1,15 @@ +月曜日 月 +月曜 月 +火曜日 火 +火曜 火 +水曜日 水 +水曜 水 +木曜日 木 +木曜 木 +金曜日 金 +金曜 金 +土曜日 土 +土曜 土 +日曜日 日 +日曜 日 + diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/day.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/day.tsv new file mode 100644 index 000000000..32794ad56 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/day.tsv @@ -0,0 +1,31 @@ +一 1 +二 2 +三 3 +四 4 +五 5 +六 6 +七 7 +八 8 +九 9 +十 10 +十一 11 +十二 12 +十三 13 +十四 14 +十五 15 +十六 16 +十七 17 +十八 18 +十九 19 +二十 20 +二十一 21 +二十二 22 +二十三 23 +二十四 24 +二十五 25 +二十六 26 +二十七 27 +二十八 28 +二十九 29 +三十 30 +三十一 31 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/months.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/months.tsv new file mode 100644 index 000000000..ab397ba20 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/months.tsv @@ -0,0 +1,12 @@ +一 1 +二 2 +三 3 +四 4 +五 5 +六 6 +七 7 +八 8 +九 9 +十 10 +十一 11 +十二 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/__init__.py new file mode 100644 index 000000000..ddf965368 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/digit.tsv new file mode 100644 index 000000000..b3f321c23 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +一 1 +二 2 +三 3 +四 4 +五 5 +六 6 +七 7 +八 8 +九 9 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/numbers/teen.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/teen.tsv new file mode 100644 index 000000000..1fccc5930 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/teen.tsv @@ -0,0 +1,26 @@ +十 10 +十一 11 +十二 12 +十三 13 +十四 14 +十五 15 +十六 16 +十七 17 +十八 18 +十九 19 +じゅう 10 +じゅっ 10 +じゅういち 11 +じゅうに 12 +じゅうさん 13 +じゅうよん 14 +じゅうし 14 +じゅうご 15 +じゅうろく 16 +じゅうろっ 16 +じゅうなな 17 +じゅうしち 17 +じゅうはち 18 +じゅうはっ 18 +じゅうきゅう 19 +じゅうく 19 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/numbers/ties.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/ties.tsv new file mode 100644 index 000000000..21c1e8a23 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/ties.tsv @@ -0,0 +1,30 @@ +二十 2 +三十 3 +四十 4 +五十 5 +六十 6 +七十 7 +八十 8 +九十 9 +にじゅう 2 +にじゅっ 2 +さんじゅう 3 +さんじゅっ 3 +よんじゅう 4 +よんじゅっ 4 +しじゅう 4 +しじゅっ 4 +ごじゅう 5 +ごじゅっ 5 +ろくじゅう 6 +ろくじゅっ 6 +ななじゅう 7 +ななじゅっ 7 +しちじゅう 7 +しちじゅっ 7 +はちじゅう 8 +はちじゅっ 8 +きゅうじゅう 9 +きゅうじゅっ 9 +くじゅう 9 +くじゅっ 9 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv new file mode 100644 index 000000000..147ef1da5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv @@ -0,0 +1,6 @@ +ゼロ 0 +零 0 +れい 0 +○ 0 +〇 0 +丸 0 diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv new file mode 100644 index 000000000..8a5eced19 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv @@ -0,0 +1,25 @@ +一 1 +二 2 +三 3 +四 4 +五 5 +六 6 +七 7 +八 8 +九 9 +十 10 +十一 11 +十二 12 +十三 13 +十四 14 +十五 15 +十六 16 +十七 17 +十八 18 +十九 19 +二十 20 +二十一 21 +二十二 22 +二十三 23 +二十四 24 +零 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/time_minutes_seconds.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/time_minutes_seconds.tsv new file mode 100644 index 000000000..48c7ea5e4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/time_minutes_seconds.tsv @@ -0,0 +1,60 @@ +一 1 +二 2 +三 3 +四 4 +五 5 +六 6 +七 7 +八 8 +九 9 +十 10 +十一 11 +十二 12 +十三 13 +十四 14 +十五 15 +十六 16 +十七 17 +十八 18 +十九 19 +二十 20 +二十一 21 +二十二 22 +二十三 23 +二十四 24 +二十五 25 +二十六 26 +二十七 27 +二十八 28 +二十九 29 +三十 30 +三十一 31 +三十二 32 +三十三 33 +三十四 34 +三十五 35 +三十六 36 +三十七 37 +三十八 38 +三十九 39 +四十 40 +四十一 41 +四十二 42 +四十三 43 +四十四 44 +四十五 45 +四十六 46 +四十七 47 +四十八 48 +四十九 49 +五十 50 +五十一 51 +五二十 52 +五十三 53 +五十四 54 +五十五 55 +五十六 56 +五十七 57 +五十八 58 +五十九 59 +六十 60 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/whitelist.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/whitelist.tsv new file mode 100644 index 000000000..21aa42674 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/data/whitelist.tsv @@ -0,0 +1,234 @@ +ぺいぺい PayPay +トリプルa AAA +トリプルA AAA +にってれ 日テレ +ニッテレ 日テレ +コビッと 19 COVID-19 +シーエム cm +ゲートウェイ GW +ゲートウェイ gw +エヌジー ng +オーエル ol +ピーブイ pv +ワールドカップ w杯 +a c AC +a c l ACL +a e b AEB +a e s AES +a g x AGX +a i AI +a l u ALU +a m d AMD +a p i API +a r p ARP +a s a p ASAP +a s m c ASMC +a s r ASR +a t m ATM +a v a s AVAS +a v r c p AVRCP +a w s AWS +aws AWS +b b c BBC +b c p BCP +b e v BEV +b g a BGA +b g p BGP +b i s BIS +b s p BSP +c b c CBC +c c s CCS +c c t v CCTV +c d n CDN +c e o CEO +c e s CES +c f o CFO +c l i CLI +c m o CMO +c m s CMS +c n n CNN +c p u CPU +c s o CSO +c s p CSP +c s s CSS +c v CV +c v t CVT +d a l i DALI +d b DB +d c DC +d c d b DCDB +d d r DDR +d d o s DDoS +d g l DGL +d g x DGX +d h c p DHCP +d l s s DLSS +d n a DNA +d n n DNN +d n s DNS +d p DP +d p d t DPDT +d p u DPU +d p x DPX +e b c EBC +e b s EBS +e c two EC2 +e c c ECC +e c s ECS +e f s EFS +e g x EGX +e k s EKS +e l b ELB +e l c b ELCB +e m r EMR +e o l EOL +e v EV +f p s FPS +f s a FSA +f s t FST +f t p FTP +f t p s FTPS +f x FX +g c c GCC +g c p GCP +g c s GCS +G f n GFN +g p u GPU +g r p c GRPC +g r u GRU +g s k GSK +g t c GTC +g u i GUI +h d m i HDMI +h e v c HEVC +h g x HGX +Hmd HMD +h p c HPC +h q HQ +h s m HSM +h t HT +h t m l HTML +h t t p HTTP +h t t p s HTTPS +i o I/O +i c m p ICMP +i e c IEC +i triple e IEEE +i g m p IGMP +i g x IGX +i i s c IISc +i i t IIT +i p IP +i s c ISC +i s i ISI +i o t IoT +k v m KVM +l g a LGA +l l c LLC +l l m LLM +l l p LLP +l s t m LSTM +l t LT +l t t LTT +m b MB +m c b MCB +m c c b MCCB +m c p MCP +m d m MDM +m f a MFA +m i t MIT +m l ML +n c NC +n g c NGC +n l p NLP +n l u NLU +n m c NMC +n o NO +n t p NTP +o c i OCI +o e m OEM +o l t c OLTC +o l t p OLTP +o n n x ONNX +o v x OVX +open c l OpenCL +open g l OpenGL +p c i PCI +p c i e PCIe +p d f PDF +p h e v PHEV +p s o PSO +pi g PyG +py g PyG +q o d a QODA +q o s QOS +r c b o RCBO +r c c b RCCB +r d s RDS +r f p RFP +r l RL +r n n RNN +r o i ROI +r s s RSS +r t RT +r t p RTP +s and p S&P +s three S3 +s a e SAE +s d k SDK +s e o SEO +s f u SFU +s m p s SMPS +s m t p SMTP +s n m p SNMP +s n s SNS +s o l SOL +s p d t SPDT +s p n SPN +s p s t SPST +s q l SQL +s q s SQS +s s d SSD +s s h SSH +s s l SSL +s s o SSO +s s r SSR +s u v SUV +s w g SWG +t c p TCP +t c p i p TCP/IP +t f t p TFTP +t j TJ +t o s TOS +t p m TPM +t p m s TPMS +t p n TPN +t s l TSL +t s m c TSMC +t t s TTS +tell net Telnet +tensor r t TensorRT +tick tock Tick-Tock +tick tock app TikTok app +tick tock is TikTok is +u d p UDP +u e f i UEFI +u f i UFI +u p s UPS +u s b USB +v a a p i VAAPI +v a api VAAPI +v d i VDI +v d p a u VDPAU +v d paw VDPAU +v d pow VDPAU +v f d VFD +v lan VLAN +v p VP +v p n VPN +v c c Vcc +v d d Vdd +v o i p VoIP +w f s a WFSA +w f s t WFST +wifi Wi-Fi diff --git a/nemo_text_processing/inverse_text_normalization/ja/graph_utils.py b/nemo_text_processing/inverse_text_normalization/ja/graph_utils.py new file mode 100644 index 000000000..abcced58a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/graph_utils.py @@ -0,0 +1,292 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_NARROW_NON_BREAK_SPACE = "\u202f" +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +NEMO_NOT_ALPHA = pynini.difference(NEMO_SIGMA, NEMO_ALPHA).optimize() +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z" +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/__init__.py new file mode 100644 index 000000000..ddf965368 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/cardinal.py new file mode 100644 index 000000000..15d17f81d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/cardinal.py @@ -0,0 +1,158 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst +from nemo_text_processing.inverse_text_normalization.ja.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 二十三 -> cardinal { integer: "23" } + e.g. にじゅうさん -> cardinal { integer: "23" } + """ + + def __init__(self): + super().__init__(name="cardinal", kind="classify") + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + graph_all = graph_ties + (graph_digit | pynutil.insert("0")) | graph_teen + + hundred = pynutil.delete("百") | pynutil.delete("ひゃく") | pynutil.delete("びゃく") | pynutil.delete("ぴゃく") + hundred_alt = ( + pynini.cross("百", "1") + | pynini.cross("ひゃく", "1") + | pynini.cross("びゃく", "1") + | pynini.cross("ぴゃく", "1") + ) + graph_hundred_component = pynini.union(((graph_digit + hundred) | hundred_alt), pynutil.insert("0")) + graph_hundred_component += pynini.union( + graph_teen | pynutil.insert("00"), (graph_ties | pynutil.insert("0")) + (graph_digit | pynutil.insert("0")) + ) + + thousand = pynutil.delete("千") | pynutil.delete("せん") | pynutil.delete("ぜん") + thousand_alt = pynini.cross("千", "1") | pynini.cross("せん", "1") | pynini.cross("ぜん", "1") + graph_thousand_component = pynini.union(((graph_digit + thousand) | thousand_alt), pynutil.insert("0")) + graph_thousand_component += graph_hundred_component + graph_thousand_component = graph_thousand_component | pynutil.insert("0000") + + tenthousand = pynutil.delete("万") | pynutil.delete("まん") + graph_tenthousand_component = pynini.union(graph_digit + tenthousand, pynutil.insert("0")) + graph_tenthousand_component += graph_thousand_component + + graph_hundredthousand_component = pynini.union( + (graph_teen | ((graph_ties | pynutil.insert("0")) + (graph_digit | pynutil.insert("0")))) + tenthousand, + pynutil.insert("00"), + ) + graph_hundredthousand_component += graph_thousand_component + + graph_million_component = pynini.union(graph_hundred_component + tenthousand, pynutil.insert("000")) + graph_million_component += graph_thousand_component + + graph_tenmillion_component = pynini.union(graph_thousand_component + tenthousand, pynutil.insert("0000")) + graph_tenmillion_component += graph_thousand_component + + hundredmillion = pynutil.delete("億") | pynutil.delete("おく") + graph_hundredmillion_component = pynini.union(graph_digit + hundredmillion, pynutil.insert("0")) + graph_hundredmillion_component += graph_tenmillion_component + + graph_billion_component = pynini.union( + (graph_teen | ((graph_ties | pynutil.insert("0")) + (graph_digit | pynutil.insert("0")))) + hundredmillion, + pynutil.insert("00"), + ) + graph_billion_component += graph_tenmillion_component + + graph_tenbillion_component = pynini.union(graph_hundred_component + hundredmillion, pynutil.insert("000")) + graph_tenbillion_component += graph_tenmillion_component + + graph_hundredbillion_component = pynini.union( + graph_thousand_component + hundredmillion, pynutil.insert("0000") + ) + graph_hundredbillion_component += graph_tenmillion_component + + graph_thousandbillion_component = pynini.union( + graph_tenthousand_component + hundredmillion, pynutil.insert("00000") + ) + graph_thousandbillion_component += graph_tenmillion_component # e.g.,五万億 = 五兆 + + graph_zyumannoku = pynini.union(graph_hundredthousand_component + hundredmillion, pynutil.insert("000000")) + graph_zyumannoku += graph_tenmillion_component # 五十万億 = 五十兆 + + graph_hyakumanoku = pynini.union(graph_million_component + hundredmillion, pynutil.insert("0000000")) + graph_hyakumanoku += graph_tenmillion_component # 五百万億 = 五百兆 + + graphsenmanoku = pynini.union(graph_tenmillion_component + hundredmillion, pynutil.insert("00000000")) + graphsenmanoku += graph_tenmillion_component # 五千万億 = 五千兆 + + trillion = pynutil.delete("兆") + graph_trillion_component = pynini.union(graph_digit + trillion, pynutil.insert("0")) + graph_trillion_component += graph_hundredbillion_component + + graph_tentrillion_component = pynini.union( + (graph_teen | ((graph_ties | pynutil.insert("0")) + (graph_digit | pynutil.insert("0")))) + trillion, + pynutil.insert("00"), + ) + graph_tentrillion_component += graph_hundredbillion_component + + graph_hundredtrillion_component = pynini.union(graph_hundred_component + trillion, pynutil.insert("000")) + graph_hundredtrillion_component += graph_hundredbillion_component + + graph_thousandtrillion_component = pynini.union(graph_thousand_component + trillion, pynutil.insert("0000")) + graph_thousandtrillion_component += graph_hundredbillion_component + + graph = pynini.union( + (graph_thousandtrillion_component | graphsenmanoku), + (graph_hundredtrillion_component | graph_hyakumanoku), + (graph_tentrillion_component | graph_zyumannoku), + (graph_trillion_component | graph_thousandbillion_component), + graph_hundredbillion_component, + graph_tenbillion_component, + graph_billion_component, + graph_hundredmillion_component, + graph_tenmillion_component, + graph_million_component, + graph_hundredthousand_component, + graph_tenthousand_component, + graph_thousand_component, + graph_hundred_component, + graph_all, + graph_digit, + graph_zero, + ) + + leading_zero = ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + graph = graph @ leading_zero | graph_zero + + self.just_cardinals = graph + + optional_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("マイナス", "-")) + pynutil.insert("\"") + ) + + final_graph = ( + optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/date.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/date.py new file mode 100644 index 000000000..0e30449e8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/date.py @@ -0,0 +1,111 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ja.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g., + 一日 -> 1日 date { day: "1" } + 五から九日 -> (5~9日) date { day: "5~9" } + 一月 -> 1月 date { month: "1" } + 三から四月 -> 3~4月 date { month: "3~4" } + 一月一日 -> 1月1日 date { month: "1" day: "1" } + 七十年代 -> 70年代 date {era: "70年代" } + 七十から八十年代 -> 70~80年代 date { era: "70~80年代" } + 二十一世紀 -> 21世紀 date { era: " 21世紀" } + 二千九年 -> 2009年 date { year: "2009" } + 二十三年二月二十五日土曜日~23年2月25日(土) -> { year: "23" month: "2" day: "25" weekday: "土" } + 七月五から九日月曜日から金曜日~7月5〜9日(月〜金) -> { month: "7" day: "5~9" weekday: "月~金" } + 今年はR六 -> { year: "今年は令和6" } + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="date", kind="classify") + + cardinal = cardinal.just_cardinals + week = pynini.string_file(get_abs_path("data/date.tsv")) + day = pynini.string_file(get_abs_path("data/day.tsv")) + month = pynini.string_file(get_abs_path("data/months.tsv")) + + day_component = ( + pynutil.insert("day: \"") + + cardinal + + pynini.closure((pynini.cross("から", "〜") + day), 0, 1) + + pynutil.delete("日") + + pynutil.insert("\"") + ) + month_component = ( + pynutil.insert("month: \"") + + cardinal + + pynini.closure((pynini.cross("から", "〜") + month), 0, 1) + + pynutil.delete("月") + + pynutil.insert("\"") + ) + year_component = ( + pynutil.insert("year: \"") + + cardinal + + pynini.closure((pynini.cross("から", "〜") + cardinal), 0, 1) + + pynutil.delete("年") + + pynutil.insert("\"") + ) + week_component = (pynutil.insert("weekday: \"(") + week + pynutil.insert(")\"")) | ( + pynutil.insert("weekday: \"(") + week + pynini.cross("から", "〜") + week + pynutil.insert(")\"") + ) + graph_era = ( + pynutil.insert("era: \"") + + cardinal + + pynini.closure((pynini.cross("から", "〜") + cardinal), 0, 1) + + (pynini.accep("年代") | pynini.accep("世紀")) + + pynutil.insert("\"") + ) + + graph_component = day_component | month_component | year_component | graph_era | week_component + graph_date = ( + pynini.closure(graph_era, 0, 1) + + pynini.closure(year_component, 0, 1) + + pynini.closure(pynutil.insert(" ") + month_component, 0, 1) + + pynini.closure(pynutil.insert(" ") + day_component, 0, 1) + ) + graph_date = graph_date | (graph_date + pynini.closure(pynutil.insert(" ") + week_component, 0, 1)) + + # specific context for era year, e.g., L6 -> "令和6年" + context = pynini.union( + pynini.accep("今年は"), + pynini.accep("来年は"), + pynini.accep("再来年は"), + pynini.accep("去年は"), + pynini.accep("一昨年は"), + pynini.accep("おととしは"), + ) + era_year = pynini.union( + pynini.cross("R", "令和"), + pynini.cross("H", "平成"), + pynini.cross("S", "昭和"), + pynini.cross("T", "大正"), + pynini.cross("M", "明治"), + ) + graph_era_year = context + era_year + cardinal + graph_content_specific = pynutil.insert("year: \"") + graph_era_year + pynutil.insert("\"") + + final_graph = graph_component | graph_date | graph_content_specific + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/decimal.py new file mode 100644 index 000000000..6e070231c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/decimal.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ja.utils import get_abs_path + + +def get_quantity(decimal): + suffix = pynini.union("万", "億", "兆") + numbers = decimal + res = numbers + pynutil.insert(' quantity: "') + suffix + pynutil.insert('"') + + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal + e.g. 一点五 -> decimnl { integer_part: "1" fractional_part: "5" } + e.g. 一点五万 -> decimal { integer_part: "1" fractional_part: "5" quantity: "万" } + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="decimal", kind="classify") + + cardinals = cardinal.just_cardinals + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + after_decimal = pynini.closure(graph_zero | graph_digit) + + decimal_point = pynutil.delete("点") + fractional_component = pynutil.insert("fractional_part: \"") + after_decimal + pynutil.insert("\"") + integer_component = pynutil.insert("integer_part: \"") + cardinals + pynutil.insert("\"") + + graph_decimal_regular = integer_component + decimal_point + pynutil.insert(" ") + fractional_component + graph_deicimal_larger = get_quantity(graph_decimal_regular) + + self.decimal = graph_decimal_regular | graph_deicimal_larger + self.just_decimal = cardinals + pynini.cross("点", ".") + after_decimal + + graph_sign = ( + pynutil.insert("negative: \"") + (pynini.cross("マイナス", "-") | pynini.accep("-")) + pynutil.insert("\"") + ) + + final_graph = ( + (graph_sign + pynutil.insert(" ") + graph_decimal_regular) + | (graph_sign + pynutil.insert(" ") + graph_deicimal_larger) + | graph_decimal_regular + | graph_deicimal_larger + ) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py new file mode 100644 index 000000000..0ced0c679 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py @@ -0,0 +1,105 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst + + +class FractionFst(GraphFst): + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + """ + Fitite state transducer for classifying fractions + e.g., + 四分の三 -> fraction { denominator: "4" numerator: "3" } + 一と四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + 一荷四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + ルート三分の一 -> fraction { denominator: "√3" numerator: "1" } + 一点六五分の五十 -> fraction { denominator: "1.65" numerator: "50" } + 二ルート六分の三 -> -> fraction { denominator: "2√6 " numerator: "3" } + """ + super().__init__(name="fraction", kind="classify") + + cardinal = cardinal.just_cardinals + decimal = decimal.just_decimal + + fraction_word = ( + pynutil.delete("分の") | pynutil.delete(" 分 の　") | pynutil.delete("分 の　") | pynutil.delete("分 の") + ) + + integer_word = pynutil.delete("と") | pynutil.delete("荷") + root_word = pynini.accep("√") | pynini.cross("ルート", "√") + + graph_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("マイナス", "-")) + pynutil.insert("\"") + ) + + graph_integer = ( + pynutil.insert("integer_part: \"") + + ( + (decimal | decimal + integer_word) + | ((cardinal + root_word + cardinal) | (cardinal + root_word + cardinal + integer_word)) + | ((root_word + cardinal) | (root_word + cardinal + integer_word)) + | (cardinal | (cardinal + integer_word)) + ) + + pynutil.insert("\"") + ) + + graph_denominator = ( + pynutil.insert("denominator: \"") + + ( + ((decimal) | (cardinal + root_word + cardinal) | (root_word + cardinal) | cardinal) + + pynini.closure(pynutil.delete(' '), 0, 1) + ) + + pynutil.insert("\"") + ) + + graph_numerator = ( + pynutil.insert("numerator: \"") + + ( + pynini.closure(pynutil.delete(' ')) + + ((decimal) | (cardinal + root_word + cardinal) | (root_word + cardinal) | cardinal) + ) + + pynutil.insert("\"") + ) + + graph_fraction_sign = ( + graph_sign + + pynutil.insert(" ") + + graph_denominator + + pynutil.insert(" ") + + fraction_word + + graph_numerator + ) + + graph_fraction_no_sign = graph_denominator + pynutil.insert(" ") + fraction_word + graph_numerator + + graph_regular_fractions = graph_fraction_sign | graph_fraction_no_sign + + graph_integer_fraction_sign = ( + pynini.closure((graph_sign + pynutil.insert(" ")), 0, 1) + + pynutil.add_weight(graph_integer, 1.1) + + pynutil.insert(" ") + + graph_denominator + + pynutil.insert(" ") + + fraction_word + + graph_numerator + ) + + final_graph = graph_regular_fractions | graph_integer_fraction_sign + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/ordinal.py new file mode 100644 index 000000000..1f48bc273 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/ordinal.py @@ -0,0 +1,40 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 第二十三 -> cardinal { morphsyntactic_feature: "第" integer: "23" } + e.g. 百番目 -> cardinal { integer: "100" morphsyntactic_feature:"番目" } + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="ordinal", kind="classify") + + cardinals = cardinal.just_cardinals + ordinals = pynini.accep("第") | pynini.accep("番目") + + integer_component = ( + pynutil.insert("integer: \"") + ((cardinals + ordinals) | (ordinals + cardinals)) + pynutil.insert("\"") + ) + + final_graph = self.add_tokens(integer_component) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py new file mode 100644 index 000000000..26e053334 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py @@ -0,0 +1,51 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class PreProcessorFst(GraphFst): + ''' + Preprocessing of TN: + 1. interjections removal such as '啊, 呃' + 2. fullwidth -> halfwidth char conversion + 好啊 -> 好 + 呃对 -> 对 + ： -> : + ； -> ; + ''' + + def __init__( + self, + remove_interjections: bool = True, + fullwidth_to_halfwidth: bool = True, + ): + super().__init__(name="PreProcessor", kind="processor") + + graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + + if remove_interjections: + remove_interjections_graph = pynutil.delete(pynini.string_file(get_abs_path('data/denylist/denylist.tsv'))) + graph @= pynini.cdrewrite(remove_interjections_graph, '', '', NEMO_SIGMA) + + if fullwidth_to_halfwidth: + fullwidth_to_halfwidth_graph = pynini.string_file(get_abs_path('data/char/fullwidth_to_halfwidth.tsv')) + graph @= pynini.cdrewrite(fullwidth_to_halfwidth_graph, '', '', NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/punctuation.py new file mode 100644 index 000000000..329db63d9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/punctuation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&'()*+,-./:;<=>?@^_`{|}~。，；：《》“”·~【】！？、‘’.<>-——_、。.「」『』‘`／・；’”“”‷･〔〕々〃ゝゞヽ〲〱〳〴〵ヾ〆，" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/time.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/time.py new file mode 100644 index 000000000..20ff3f34a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/time.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst, insert_space +from nemo_text_processing.inverse_text_normalization.ja.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifing time, e.g., + 一時三十分 -> time { hours: "1" minutes: "0" } + 五時二十分過ぎ -> time { hours: "5" minutes: "20" suffix: "過ぎ" } + 八時半頃 -> time { hours: "8" minutes: "半" suffix: "頃" } + 十時五分前 -> time { hours: "10" minutes: "25" suffix: "前" } + 正午一分前 -> time { hours: "正午" minutes: "1" suffix: "前" } + 正午十分過ぎ -> time { hours: "正午" minutes: "10" suffix: "過ぎ" } + """ + + def __init__(self): + super().__init__(name="time", kind="classify") + + hours = pynini.string_file(get_abs_path("data/time_hours.tsv")) + minutes_seconds = pynini.string_file(get_abs_path("data/time_minutes_seconds.tsv")) + + hour_component = ( + pynutil.insert("hours: \"") + + ((hours + pynutil.delete("時")) | pynini.accep("正午")) + + pynutil.insert("\"") + ) + minute_component = ( + pynutil.insert("minutes: \"") + + ((minutes_seconds + pynutil.delete("分")) | pynini.accep("半")) + + pynutil.insert("\"") + ) + second_component = ( + pynutil.insert("seconds: \"") + minutes_seconds + pynutil.delete("秒") + pynutil.insert("\"") + ) + + graph_regular = ( + pynini.closure(hour_component + insert_space + minute_component + insert_space + second_component) + | pynini.closure(hour_component | minute_component | second_component) + | pynini.closure(hour_component + insert_space + minute_component) + | pynini.closure(minute_component + insert_space + second_component) + ) + + words = pynini.accep("前") | pynini.accep("過ぎ") | pynini.accep("頃") + suffix = pynutil.insert("suffix: \"") + words + pynutil.insert("\"") + graph = graph_regular + pynini.closure(insert_space + suffix) + + final_graph = graph + + final_graph = self.add_tokens(final_graph.optimize()) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..849c3c2c9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py @@ -0,0 +1,110 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_SIGMA, + GraphFst, + generator_main, +) +from nemo_text_processing.inverse_text_normalization.ja.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.ja.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str = INPUT_LOWER_CASED, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"jp_itn_{input_case}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + ordinal_graph = ordinal.fst + + date = DateFst(cardinal) + date_graph = date.fst + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + fraction = FractionFst(cardinal, decimal) + fraction_graph = fraction.fst + + time = TimeFst() + time_graph = time.fst + + word_graph = WordFst().fst + whitelist_graph = WhiteListFst().fst + punct_graph = PunctuationFst().fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(cardinal_graph, 1.0) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.0) + | pynutil.add_weight(time_graph, 1.0) + | pynutil.add_weight(word_graph, 100) + | pynutil.add_weight(punct_graph, 1.1) + ) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") + tagger = pynini.closure(token, 1) + + self.fst = tagger + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/whitelist.py new file mode 100644 index 000000000..5c8b266a0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/whitelist.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ja.utils import get_abs_path + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + """ + + def __init__(self): + super().__init__(name="whitelist", kind="classify") + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + graph = (pynutil.insert('name: "')) + (whitelist) + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/word.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/word.py new file mode 100644 index 000000000..0560d4252 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/word.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. sleep -> tokens { name: "sleep" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/utils.py b/nemo_text_processing/inverse_text_normalization/ja/utils.py new file mode 100644 index 000000000..fd3017d28 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/utils.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + <<<<<<< HEAD + <<<<<<< HEAD + + ======= + + >>>>>>> 0a4a21c (Jp itn 20240221 (#141)) + ======= + + >>>>>>> 59f46198ab4c8880c6a5fb88f3cbee9530156498 + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/__init__.py new file mode 100644 index 000000000..ddf965368 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py new file mode 100644 index 000000000..62d41cb65 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + GraphFst, + delete_space, +) + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals, e.g. + cardinal { integer: "23" } -> 23 + cardinal { negative: "-" integer: "23" } -> -23 + cardinal { positive: "+" integer: "23" } -> +23 + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + optional_sign = ( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + delete_space + ) + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + exactly_three_digits = NEMO_DIGIT**3 + at_most_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) + + group_by_threes = at_most_three_digits + (pynutil.insert(",") + exactly_three_digits).closure() + graph = graph @ group_by_threes + + final_graph = pynini.closure(optional_sign, 0, 1) + graph + + final_graph = self.delete_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/date.py new file mode 100644 index 000000000..b765b338f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/date.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g., + date { day: "1" } -> 1日 + date { day: "5~9" } -> 5~9日 + date { month: "1" } -> 1月 + date { month: "3~4" } -> 3~4月 + date { month: "1" day: "1" } ->1月1日 + date { era: "70年代" } -> 70年代 + date { era: "70~80年代" } -> 70~80年代 + date { era: " 21世紀" } -> 21世紀 + date { year: "2009" } -> 2009年 + date { year: "23" month: "2" day: "25" weekday: "土" } -> 23年2月25日(土) + date { month: "7" day: "5~9" weekday: "月~金" } -> 月5〜9日(月〜金) + date { { year: "今年は令和6" } } -> 今年は令和6年 + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + + day_component = ( + pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.insert("日") + pynutil.delete("\"") + ) + month_component = ( + pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.insert("月") + pynutil.delete("\"") + ) + year_component = ( + pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.insert("年") + pynutil.delete("\"") + ) + era_component = pynutil.delete("era: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + weekday_component = pynutil.delete("weekday: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_regular = ( + pynini.closure(era_component) + + pynini.closure(pynutil.delete(" ") + year_component, 0, 1) + + pynini.closure(pynutil.delete(" ") + month_component, 0, 1) + + pynini.closure(pynutil.delete(" ") + day_component, 0, 1) + + pynini.closure(pynutil.delete(" ") + weekday_component, 0, 1) + ) + graph = graph_regular | era_component + + final_graph = self.delete_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py new file mode 100644 index 000000000..bb2af393f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal + e.g. decimal { integer_part: "1" fractional_part: "5" } -> 1.5 + e.g. decimal { integer_part: "1" fractional_part: "5" quantity: "万" } -> 1.5万 + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + + decimal_point = pynutil.insert(".") + integer_component = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + fractional_component = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + ) + quantity_component = pynutil.delete("quantity: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_decimal = integer_component + decimal_point + pynutil.delete(" ") + fractional_component + graph_decimal_larger = ( + integer_component + + decimal_point + + pynutil.delete(" ") + + fractional_component + + pynutil.delete(" ") + + quantity_component + ) + + graph_sign = pynutil.delete("negative: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph = ( + graph_decimal + | graph_decimal_larger + | (graph_sign + pynutil.delete(" ") + graph_decimal) + | (graph_sign + pynutil.delete(" ") + graph_decimal_larger) + ) + + final_graph = self.delete_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py new file mode 100644 index 000000000..ee320ceb4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import ( + NEMO_NON_BREAKING_SPACE, + NEMO_NOT_QUOTE, + GraphFst, +) + + +class FractionFst(GraphFst): + def __init__(self): + """ + Fitite state transducer for classifying fractions + e.g., + fraction { denominator: "4" numerator: "3" } -> 3/4 + fraction { integer: "1" denominator: "4" numerator: "3" } -> 1 3/4 + fraction { integer: "1" denominator: "4" numerator: "3" } -> 1 3/4 + fraction { denominator: "√3" numerator: "1" } -> 1/√3 + fraction { denominator: "1.65" numerator: "50" } -> 50/1.65 + fraction { denominator: "2√6" numerator: "3" } -> 3/2√6 + """ + super().__init__(name="fraction", kind="verbalize") + + sign_component = pynutil.delete("negative: \"") + pynini.closure("-", 1) + pynutil.delete("\"") + + integer_component = ( + pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + denominator_component = ( + pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + numerator_component = ( + pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + regular_graph = ( + pynini.closure((sign_component + pynutil.delete(" ")), 0, 1) + + pynini.closure(integer_component + pynutil.delete(" ") + pynutil.insert(NEMO_NON_BREAKING_SPACE)) + + numerator_component + + pynutil.delete(" ") + + pynutil.insert("/") + + denominator_component + ) + + final_graph = self.delete_tokens(regular_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py new file mode 100644 index 000000000..ae1e37759 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. ordinal { morphsyntactic_feature: "第" integer: "23" } -> 第23 + e.g. ordinal { integer: "100" morphsyntactic_feature: "番目" } -> 100番目 + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + + integer_component = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + final_graph = self.delete_tokens(integer_component) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/post_processing.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/post_processing.py new file mode 100644 index 000000000..80a951958 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/post_processing.py @@ -0,0 +1,132 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import ( + NEMO_DIGIT, + NEMO_NARROW_NON_BREAK_SPACE, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + generator_main, +) +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "zh_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def set_punct_dict(self): + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '՚', + '՝', + 'י', + '׳', + 'ߴ', + 'ߵ', + 'ᑊ', + 'ᛌ', + '᾽', + '᾿', + '`', + '´', + '῾', + '‘', + '’', + '‛', + '′', + '‵', + 'ꞌ', + '＇', + '｀', + '𖽑', + '𖽒', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + """ + + apply_narrow_space = pynini.cdrewrite( + pynini.cross(NEMO_SPACE, NEMO_NARROW_NON_BREAK_SPACE), + NEMO_DIGIT, + (pynini.closure(NEMO_DIGIT, 1) + pynini.accep("/") + pynini.closure(NEMO_DIGIT, 1)), + NEMO_SIGMA, + ) + # converting space between digit and digit/digit to narow space + delete_regular_space = pynini.cdrewrite(pynutil.delete(NEMO_SPACE), NEMO_NOT_SPACE, NEMO_NOT_SPACE, NEMO_SIGMA) + # deleting all normal spaces + reapply_regular_space = pynini.cdrewrite( + pynini.cross(NEMO_NARROW_NON_BREAK_SPACE, NEMO_SPACE), + NEMO_DIGIT, + (pynini.closure(NEMO_DIGIT, 1) + pynini.accep("/") + pynini.closure(NEMO_DIGIT, 1)), + NEMO_SIGMA, + ) + # convert narrow space to normal space + + remove_space_around_single_quote = apply_narrow_space @ delete_regular_space @ reapply_regular_space + + # this works if spaces in between (good) + # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA + + graph = remove_space_around_single_quote.optimize() + + return graph diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/postprocessor.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/postprocessor.py new file mode 100644 index 000000000..103cfb7a8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/postprocessor.py @@ -0,0 +1,81 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil, utf8 + +from nemo_text_processing.text_normalization.zh.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_PUNCT, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, +) +from nemo_text_processing.text_normalization.zh.utils import get_abs_path + + +class PostProcessor(GraphFst): + ''' + Postprocessing of TN, now contains: + 1. punctuation removal + 2. letter case conversion + 3. oov tagger + ''' + + def __init__( + self, + remove_puncts: bool = False, + to_upper: bool = False, + to_lower: bool = False, + tag_oov: bool = False, + ): + super().__init__(name="PostProcessor", kind="processor") + + graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + if remove_puncts: + remove_puncts_graph = pynutil.delete( + pynini.union(NEMO_PUNCT, pynini.string_file(get_abs_path('data/char/punctuations_zh.tsv'))) + ) + graph @= pynini.cdrewrite(remove_puncts_graph, "", "", NEMO_SIGMA).optimize() + + if to_upper or to_lower: + if to_upper: + conv_cases_graph = pynini.inverse(pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv'))) + else: + conv_cases_graph = pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv')) + + graph @= pynini.cdrewrite(conv_cases_graph, "", "", NEMO_SIGMA).optimize() + + if tag_oov: + zh_charset_std = pynini.string_file(get_abs_path("data/char/charset_national_standard_2013_8105.tsv")) + zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv")) + + zh_charset = ( + zh_charset_std | zh_charset_ext | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv")) + ) + en_charset = NEMO_DIGIT | NEMO_ALPHA | NEMO_PUNCT | NEMO_WHITE_SPACE + charset = zh_charset | en_charset + + with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f: + tags = f.readline().strip().split('\t') + assert len(tags) == 2 + ltag, rtag = tags + + oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset) + tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag) + graph @= pynini.cdrewrite(tag_oov_graph, "", "", NEMO_SIGMA).optimize() + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/time.py new file mode 100644 index 000000000..8e95e14cf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/time.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g., + time { hours: "1" minutes: "0" } -> 1時30分 -> + time { hours: "5" minutes: "20" suffix: "過ぎ" } -> 5時20分 + time { hours: "8" minutes: "半" suffix: "頃" } -> 8時半頃 + time { hours: "10" minutes: "25" suffix: "前" } -> 10時5分前 + time { hours: "正午" minutes: "1" suffix: "前" } -> 正午1分前 + time { hours: "正午" minutes: "10" suffix: "過ぎ" } -> 正午10分過ぎ + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + + hours_component = ( + pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.insert("時") + pynutil.delete("\"") + ) + hours_component_alt = pynutil.delete("hours: \"") + pynini.accep("正午") + pynutil.delete("\"") + hours_component |= hours_component_alt + + minutes_component = ( + pynutil.delete("minutes: \"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.insert("分") + + pynutil.delete("\"") + ) + minutes_component_alt = pynutil.delete("minutes: \"") + pynini.accep("半") + pynutil.delete("\"") + minutes_component |= minutes_component_alt + second_component = ( + pynutil.delete("seconds: \"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.insert("秒") + + pynutil.delete("\"") + ) + suffix_component = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_regular_time = ( + pynini.closure(hours_component) + + pynini.closure(delete_space + minutes_component) + + pynini.closure(delete_space + second_component) + + pynini.closure(delete_space + suffix_component) + ) + graph = graph_regular_time + + final_graph = graph + + final_graph = self.delete_tokens(final_graph.optimize()) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py new file mode 100644 index 000000000..90f8089ee --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py @@ -0,0 +1,58 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst() + ordinal_graph = ordinal.fst + + decimal = DecimalFst() + decimal_graph = decimal.fst + + fraction = FractionFst() + fraction_graph = fraction.fst + + date = DateFst() + date_graph = date.fst + + time = TimeFst() + time_graph = time.fst + + whitelist_graph = WhiteListFst().fst + graph = ( + cardinal_graph | date_graph | time_graph | ordinal_graph | decimal_graph | fraction_graph | whitelist_graph + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py new file mode 100644 index 000000000..7624d5f1b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py @@ -0,0 +1,59 @@ +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst, delete_space, generator_main +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.postprocessor import PostProcessor +from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst + +# from nemo.utils import logging + + +class VerbalizeFinalFst(GraphFst): + """ """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ja_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + else: + # token_graph = VerbalizeFst(deterministic=deterministic) + token_graph = VerbalizeFst().fst + # token_verbalizer = ( + # pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") + # ) + token_verbalizer = ( + pynutil.delete("tokens {") + delete_space + token_graph + delete_space + pynutil.delete(" }") + ) + verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space) + + postprocessor = PostProcessor( + remove_puncts=False, + to_upper=False, + to_lower=False, + tag_oov=False, + ) + + self.fst = (verbalizer @ postprocessor.fst).optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py new file mode 100644 index 000000000..1c21ce8d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WhiteListFst(GraphFst): + ''' + tokens { whitelist: "ATM" } -> A T M + ''' + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + + whitelist = pynutil.delete("name: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + graph = whitelist + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/math_symbol.py b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/word.py similarity index 77% rename from nemo_text_processing/text_normalization/zh/verbalizers/math_symbol.py rename to nemo_text_processing/inverse_text_normalization/ja/verbalizers/word.py index 924f9b5ab..7e5e7b91c 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/math_symbol.py +++ b/nemo_text_processing/inverse_text_normalization/ja/verbalizers/word.py @@ -11,19 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + -class MathSymbol(GraphFst): +class WordFst(GraphFst): ''' - tokens { sign: "加" } -> 加 + tokens { name: "一" } -> 一 ''' def __init__(self, deterministic: bool = True, lm: bool = False): - super().__init__(name="sign", kind="verbalize", deterministic=deterministic) + super().__init__(name="word", kind="verbalize", deterministic=deterministic) - graph = pynutil.delete('score: \"') + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('\"') + graph = pynutil.delete("name: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"") self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/__init__.py new file mode 100644 index 000000000..f541211af --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/data/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/currency.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/currency.tsv new file mode 100644 index 000000000..fd2127530 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/currency.tsv @@ -0,0 +1,9 @@ +달러 $ +불 $ +유로 € +엔 ¥ +파운드 £ +위안 ¥ +페소 $ +루피 ₹ +원 ₩ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/measure_units.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/measure_units.tsv new file mode 100644 index 000000000..ff79b8e91 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/measure_units.tsv @@ -0,0 +1,61 @@ +킬로미터 km +미터 m +센티미터 cm +밀리미터 mm +마이크로미터 μm +나노미터 nm +킬로그램 kg +그램 g +톤 t +밀리그램 mg +마이크로그램 μg +리터 L +밀리리터 ml +씨씨 cc +시간 h +분 min +초 s +뉴턴 N +와트 W +킬로와트 kW +킬로와트시 kWh +헤르츠 Hz +킬로헤르츠 kHz +메가헤르츠 MHz +기가헤르츠 GHz +도 ° +퍼센트 % +프로 % +분당회전수 rpm +알피엠 rpm +볼트 V +밀리볼트 mV +킬로볼트 kV +암페어 A +밀리암페어 mA +평 py +제곱미터 m² +제곱킬로미터 km² +제곱센티미터 cm² +세제곱미터 m³ +기가바이트 GB +기가 GB +테라바이트 TB +테라 TB +메가바이트 MB +메가 MB +킬로바이트 KB +바이트 B +비트 bit +칼로리 cal +킬로칼로리 kcal +줄 J +킬로줄 kJ +마력 hp +옴 Ω +파스칼 Pa +헥토파스칼 hPa +데시벨 dB +루멘 lm +럭스 lx +픽셀 px \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/months.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/months.tsv new file mode 100644 index 000000000..52039ef35 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/months.tsv @@ -0,0 +1,12 @@ +일 1 +이 2 +삼 3 +사 4 +오 5 +유 6 +칠 7 +팔 8 +구 9 +시 10 +십일 11 +십이 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/data/numbers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/numbers/digit.tsv new file mode 100644 index 000000000..9871cb9cf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +일 1 +이 2 +삼 3 +사 4 +오 5 +육 6 +칠 7 +팔 8 +구 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/numbers/zero.tsv new file mode 100644 index 000000000..cbf967001 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/numbers/zero.tsv @@ -0,0 +1 @@ +영 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/counter_suffix.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/counter_suffix.tsv new file mode 100644 index 000000000..e240760ed --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/counter_suffix.tsv @@ -0,0 +1,17 @@ +개 +명 +병 +마리 +대 +송이 +포기 +사람 +자루 +채 +켤레 +그루 +벌 +잔 +장 +권 +살 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/digit.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/digit.tsv new file mode 100644 index 000000000..d2fdd1846 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/ordinals/digit.tsv @@ -0,0 +1,9 @@ +한 1 +두 2 +세 3 +네 4 +다섯 5 +여섯 6 +일곱 7 +여덟 8 +아홉 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/time/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/data/time/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/time/ten_prefix.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/time/ten_prefix.tsv new file mode 100644 index 000000000..fe2800413 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/time/ten_prefix.tsv @@ -0,0 +1,4 @@ +이 2 +삼 3 +사 4 +오 5 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/time/time_hours.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/time/time_hours.tsv new file mode 100644 index 000000000..8044e4006 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/time/time_hours.tsv @@ -0,0 +1,23 @@ +한 1 +두 2 +세 3 +네 4 +다섯 5 +여섯 6 +일곱 7 +여덟 8 +아홉 9 +열 10 +열한 11 +열두 12 +열세 13 +열네 14 +열다섯 15 +열여섯 16 +열일곱 17 +열여덟 18 +열아홉 19 +스무 20 +스물한 21 +스물두 22 +스물세 23 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/data/whitelist.tsv b/nemo_text_processing/inverse_text_normalization/ko/data/whitelist.tsv new file mode 100644 index 000000000..5ff302fb6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/data/whitelist.tsv @@ -0,0 +1,35 @@ +박사 Dr. +박사 dr. +씨 Mr. +씨 mr. +양 Ms. +양 ms. +여사 Mrs. +여사 mrs. +산 mt. +산 Mt. +교수 Prof. +교수 prof. +시니어 sr. +시니어 Sr. +주니어 jr. +주니어 Jr. +대로 Ave. +대로 ave. +번호 no. +번호 No. +왼쪽 괄호 ( +오른쪽 괄호 ) +더하기 + +마이너스 - +시그마 Σ +에타 η +카파 κ +오메가 ω +시그마 σ +알파 α +뉴 ν +델타 δ +이오타 ι +박사학위 Ph.D. +등 etc. \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ko/graph_utils.py b/nemo_text_processing/inverse_text_normalization/ko/graph_utils.py new file mode 100644 index 000000000..50f1eb3b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/graph_utils.py @@ -0,0 +1,292 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_NARROW_NON_BREAK_SPACE = "\u202f" +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +NEMO_NOT_ALPHA = pynini.difference(NEMO_SIGMA, NEMO_ALPHA).optimize() +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z" +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/cardinal.py new file mode 100644 index 000000000..8befd4250 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/cardinal.py @@ -0,0 +1,118 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 마이너스 이십삼 -> cardinal { integer: "23" negative: "-" } } + + Args: + input_case: accepting Korean input. + """ + + def __init__(self): + super().__init__(name="cardinal", kind="classify") + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + + ten = pynutil.delete("십") + ten_alt = pynini.cross("십", "1") + ### Responsible for second digit of two digit number. ex) 20's 2 + graph_ten_component = pynini.union((graph_digit + ten) | ten_alt, pynutil.insert("0")) + ### Responsible for the first digit of number. ex) 1,2,3,4,5,,, + graph_ten_component += graph_digit | pynutil.insert("0") + + hundred = pynutil.delete("백") + hundred_alt = pynini.cross("백", "1") + graph_hundred_component = pynini.union(((graph_digit + hundred) | hundred_alt), pynutil.insert("0")) + graph_hundred_component += graph_ten_component + + thousand = pynutil.delete("천") + thousand_alt = pynini.cross("천", "1") + graph_thousand_component = pynini.union(((graph_digit + thousand) | thousand_alt), pynutil.insert("0")) + graph_thousand_component += graph_hundred_component + + # "만" marks the 10,000 unit. + # It shifts the number by four digits (Korean units grow in 4-digit groups). + tenthousand = pynutil.delete("만") + tenthousand_alt = pynini.cross("만", "1") # "만"을 leading 1로 취급 + + # thousand_component가 "0"만 출력하는 케이스를 막고 싶으면(선택) + thousand_input = pynini.project(graph_thousand_component, "input").optimize() + thousand_input_nonempty = pynini.difference(thousand_input, pynini.accep("")).optimize() + graph_thousand_component_nonempty = (thousand_input_nonempty @ graph_thousand_component).optimize() + + # Handle the "만" unit (10,000). + # Korean numbers increase by 4-digit units, so "만" shifts the value by four digits. + # Supports patterns like 만, 만, and 만. + graph_tenthousand_component = pynini.union( + (graph_thousand_component + tenthousand) + graph_thousand_component, + tenthousand_alt + pynutil.insert("0000"), + # "만" + <1~9999> + tenthousand_alt + graph_thousand_component_nonempty, + # implicit leading part: <0000> + <0~9999> + pynutil.insert("0000") + graph_thousand_component, + ).optimize() + hundredmillion = pynutil.delete("억") + hundredmillion_alt = pynini.cross("억", "1") + graph_hundredmillion_component = pynini.union( + ((graph_thousand_component + hundredmillion) | hundredmillion_alt), pynutil.insert("0000") + ) + graph_hundredmillion_component += graph_tenthousand_component + + trillion = pynutil.delete("조") + trillion_alt = pynini.cross("조", "1") + graph_trillion_component = pynini.union( + ((graph_thousand_component + trillion) | trillion_alt), pynutil.insert("0000") + ) + graph_trillion_component += graph_hundredmillion_component + + tenquadrillion = pynutil.delete("경") + tenquadrillion_alt = pynini.cross("경", "1") + graph_tenquadrillion_component = pynini.union( + ((graph_thousand_component + tenquadrillion) | tenquadrillion_alt), pynutil.insert("0000") + ) + graph_tenquadrillion_component += graph_trillion_component + + graph = pynini.union( + ### From biggest unit to smallest, everything is included + graph_tenquadrillion_component + | graph_zero + ) + + leading_zero = ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + graph = (graph @ leading_zero) | graph_zero + + self.just_cardinals = graph + + negative_sign = pynini.closure( + (pynini.cross("마이너스", 'negative: "-"') | pynini.cross("-", 'negative: "-"')) + delete_space, 0, 1 + ) + + final_graph = ( + negative_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/date.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/date.py new file mode 100644 index 000000000..b02e80984 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/date.py @@ -0,0 +1,61 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, + e.g. 이천십이년 일월 오일 -> date { year: "2012" month: "1" day: "5" } + e.g. 오월 -> date { month: "5" } + e.g. 칠일 -> date { day: "7" } + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="date", kind="classify") + + cardinal = cardinal.just_cardinals + month = pynini.string_file(get_abs_path("data/months.tsv")) + + year_suffix = pynini.cross("년", "") + month_suffix = pynini.cross("월", "") + day_suffix = pynini.cross("일", "") + + delete_space = pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1) + between_fields = delete_space + pynutil.insert(NEMO_SPACE) + + year_component = pynutil.insert("year: \"") + cardinal + year_suffix + pynutil.insert("\"") + month_component = pynutil.insert("month: \"") + month + month_suffix + pynutil.insert("\"") + day_component = pynutil.insert("day: \"") + cardinal + day_suffix + pynutil.insert("\"") + + graph_component = year_component | month_component + + graph_date = ( + year_component + | month_component + | (year_component + between_fields + month_component) + | (month_component + between_fields + day_component) + | (year_component + between_fields + month_component + between_fields + day_component) + ) + + final_graph = graph_component | graph_date + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/decimal.py new file mode 100644 index 000000000..96cd012b3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/decimal.py @@ -0,0 +1,74 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +def get_quantity(decimal): + suffix = pynini.union("만", "억", "조", "경") + numbers = decimal + res = numbers + pynutil.insert(' quantity: "') + suffix + pynutil.insert('"') + + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal + e.g. 일점오 -> decimnl { integer_part: "1" fractional_part: "5" } + e.g. 일점오만 -> decimal { integer_part: "1" fractional_part: "5" quantity: "만" } + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="decimal", kind="classify") + + cardinals = cardinal.just_cardinals + man_as_10000 = pynini.cross("만", "10000") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + decimal_part = pynini.closure(graph_zero | graph_digit, 1) + + decimal_point = pynutil.delete("점") + integer_number = cardinals | man_as_10000 + integer_part = pynutil.insert("integer_part: \"") + integer_number + pynutil.insert("\"") + fractional_part = pynutil.insert("fractional_part: \"") + decimal_part + pynutil.insert("\"") + + graph_decimal_regular = ( + integer_part + decimal_point + pynutil.insert(NEMO_SPACE) + fractional_part + ) # Regular decimal like 1.5 + graph_deicimal_larger = get_quantity( + graph_decimal_regular + ) # If decimal is used to express big numbers like 15000 -> "1.5만" + + self.decimal = graph_decimal_regular | graph_deicimal_larger + self.just_decimal = cardinals | (cardinals + pynini.cross("점", ".") + decimal_part) + + graph_sign = ( + pynutil.insert("negative: \"") + (pynini.cross("마이너스", "-") | pynini.accep("-")) + pynutil.insert("\"") + ) + + final_graph = ( + (graph_sign + pynutil.insert(" ") + graph_decimal_regular) + | (graph_sign + pynutil.insert(" ") + graph_deicimal_larger) + | graph_decimal_regular + | graph_deicimal_larger + ) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/fraction.py new file mode 100644 index 000000000..a8f9f919d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/fraction.py @@ -0,0 +1,139 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst + + +class FractionFst(GraphFst): + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + """ + Fitite state transducer for classifying fractions + e.g., + fraction { denominator: "사" numerator: "삼" } -> 3/4 + fraction { integer_part: "일" denominator: "사" numerator: "삼" } -> 1 3/4 + fraction { denominator: "루트삼" numerator: "일" } -> 1/√3 + fraction { denominator: "일점육오" numerator: "오십" } -> 50/1.65 + fraction { denominator: "이루트육" numerator: "삼" } -> 3/2√6 + """ + super().__init__(name="fraction", kind="classify") + + cardinal = cardinal.just_cardinals + decimal = decimal.just_decimal + + # Expression between fraction. Means the dash "/" + fraction_word = pynutil.delete("분의") + # Expression combining mixed number and fraction. Optional to use + connecting_word = pynutil.delete("와") | pynutil.delete("과") + # Expression for "√" + root_word = pynini.accep("√") | pynini.cross("루트", "√") + + graph_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("마이너스", "-")) + pynutil.insert("\"") + ) + + # graph_mixed_number considers all of possible combination number you can have in front of fraction + graph_mixed_number = ( + pynutil.insert("integer_part: \"") + + ( + decimal + | (decimal + connecting_word) + | (root_word + decimal) + | (cardinal + root_word + decimal) + | (root_word + decimal + connecting_word) + | (cardinal + root_word + decimal + connecting_word) + | cardinal + | (cardinal + connecting_word) + | (root_word + cardinal) + | (cardinal + root_word + cardinal) + | (root_word + cardinal + connecting_word) + | (cardinal + root_word + cardinal + connecting_word) + ) + + pynutil.insert("\"") + ) + + graph_denominator = ( + pynutil.insert("denominator: \"") + + ( + ( + decimal + | (cardinal + root_word + decimal) + | (root_word + decimal) + | cardinal + | (cardinal + root_word + cardinal) + | (root_word + cardinal) + ) + + pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1) + ) + + pynutil.insert("\"") + ) + + graph_numerator = ( + pynutil.insert("numerator: \"") + + ( + ( + decimal + | (cardinal + root_word + decimal) + | (root_word + decimal) + | cardinal + | (cardinal + root_word + cardinal) + | (root_word + cardinal) + ) + + pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1) + ) + + pynutil.insert("\"") + ) + + graph_fraction_sign = ( + graph_sign + + pynutil.insert(NEMO_SPACE) + + graph_denominator + + pynutil.insert(NEMO_SPACE) + + fraction_word + + graph_numerator + ) + graph_fraction_no_sign = graph_denominator + pynutil.insert(NEMO_SPACE) + fraction_word + graph_numerator + # Only fraction like "1/3" or "- 1/3" + graph_fractions = graph_fraction_sign | graph_fraction_no_sign + # Mixed number fraction like "2 1/3" or "-2 1/3" + graph_mixed_number_fraction = ( + pynini.closure((graph_sign + pynutil.insert(" ")), 0, 1) + + pynutil.add_weight(graph_mixed_number, 1.1) + + pynutil.insert(NEMO_SPACE) + + graph_denominator + + pynutil.insert(NEMO_SPACE) + + fraction_word + + graph_numerator + ) + + # ---- NEW: optional josa after fraction (prevents "이"/"만" from being re-tokenized as cardinal) ---- + josa_single = pynini.union("만", "이", "가", "은", "는", "을", "를", "로", "도", "다") + josa_multi = pynini.union("부터", "까지") + josa = (josa_single | josa_multi | (josa_single + josa_multi)).optimize() + + trailing_josa = pynini.closure( + pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1) # optional space + + pynutil.insert(' suffix: "') + + josa + + pynutil.insert('"'), + 0, + 1, + ) + + final_graph = (graph_fractions | graph_mixed_number_fraction) + trailing_josa + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/measure.py new file mode 100644 index 000000000..5a2d77ced --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/measure.py @@ -0,0 +1,130 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( + NEMO_SPACE, + GraphFst, + convert_space, + delete_extra_space, +) +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. 십이 킬로그램 -> measure { cardinal { integer: "12" } units: "kg" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="measure", kind="classify") + + base_cardinal = cardinal.just_cardinals + man_as_10000 = pynini.cross("만", "10000") + cardinal_graph = base_cardinal | man_as_10000 + # Graphing fraction (extended to support root denominators like "√") + root_word = pynini.accep("√") | pynini.cross("루트", "√") + root_cardinal = (root_word + cardinal_graph).optimize() # e.g., 루트구 -> √9 + den_for_fraction = (cardinal_graph | root_cardinal).optimize() + num_for_fraction = (cardinal_graph | root_cardinal).optimize() + graph_unit = pynini.string_file(get_abs_path("data/measure_units.tsv")) + + delete_any_space = pynini.closure(pynutil.delete(NEMO_SPACE)) + # Negative sign + negative_word = pynini.union("마이너스", "영하") + graph_negative = pynini.cross(negative_word, 'negative: "true"') + delete_extra_space + optional_graph_negative = pynini.closure(graph_negative, 0, 1) + # Graphing measurement units + unit_singular = convert_space(graph_unit) + # For units that has "/", like km/h + unit_per = ( + unit_singular + + delete_any_space + + pynini.cross(pynini.union("퍼", "당"), "/") + + delete_any_space + + unit_singular + ) + + graph_unit_final = pynutil.insert('units: "') + (unit_singular | unit_per) + pynutil.insert('"') + + # Graphing decimal + graph_digit_tsv = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.cross("영", "0") | pynini.cross("공", "0") + decimal_fractional_part = pynini.closure(graph_digit_tsv | graph_zero, 1) + + graph_decimal = ( + pynutil.insert('integer_part: "') + + cardinal_graph + + pynutil.insert('"') + + delete_any_space + + pynini.cross("점", NEMO_SPACE) + + delete_any_space + + pynutil.insert('fractional_part: "') + + decimal_fractional_part + + pynutil.insert('"') + ) + + # Graphing fraction + graph_fraction = ( + pynutil.insert("fraction { ") + + pynutil.insert('denominator: "') + + den_for_fraction + + pynutil.insert('"') + + delete_any_space + + pynutil.delete("분의") + + delete_any_space + + pynutil.insert(' numerator: "') + + num_for_fraction + + pynutil.insert('"') + + pynutil.insert(" }") + ) + + final_graph_cardinal = ( + delete_any_space + + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert('integer: "') + + cardinal_graph + + pynutil.insert('"') + + pynutil.insert(" }") + + pynutil.insert(NEMO_SPACE) + + delete_any_space + + graph_unit_final + ) + + final_graph_decimal = ( + delete_any_space + + pynutil.insert("decimal { ") + + optional_graph_negative + + graph_decimal + + pynutil.insert(" }") + + pynutil.insert(NEMO_SPACE) + + delete_any_space + + graph_unit_final + ) + + final_graph_fraction = ( + delete_any_space + graph_fraction + pynutil.insert(NEMO_SPACE) + delete_any_space + graph_unit_final + ) + + final_graph = final_graph_cardinal | final_graph_decimal | final_graph_fraction + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/money.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/money.py new file mode 100644 index 000000000..3b46e9a0d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/money.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money + e.g. 오만 삼천원 -> money { integer_part: "53000" currency: "₩" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="money", kind="classify") + + decimals = decimal.just_decimal + currency = pynini.string_file(get_abs_path("data/currency.tsv")) + + man_as_10000 = pynini.cross("만", "10000") + number_for_money = decimals | man_as_10000 + + # Accepting space if there are one between integer and currency + spacing = pynini.closure(pynini.accep(NEMO_SPACE), 0, 1) + graph_integer = pynutil.insert('integer_part: "') + number_for_money + pynutil.insert('"') + spacing + graph_unit = pynutil.insert(" currency: \"") + currency + pynutil.insert("\"") + + graph_final = graph_integer + graph_unit + + final_graph = self.add_tokens(graph_final) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/ordinal.py new file mode 100644 index 000000000..094fbebde --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/ordinal.py @@ -0,0 +1,126 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, delete_space +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +def get_counter(ordinal): + # counter suffix file (개, 명, 병, 마리, ...) + suffix = pynini.string_file(get_abs_path("data/ordinals/counter_suffix.tsv")) + # allowed trailing josa (optional) to capture forms like "네개를", "여섯명만" + josa_single = pynini.union("만", "이", "가", "은", "는", "을", "를", "로", "도", "다") + josa_multi = pynini.union("부터", "까지") + josa = (josa_single | josa_multi | (josa_single + josa_multi)).optimize() + + counter_field = pynutil.insert('" counter: "') + suffix + suffix_field = pynutil.insert('" suffix: "') + josa + + return ( + ordinal + + pynini.closure(delete_space, 0, 1) + + counter_field + + pynini.closure(pynini.closure(delete_space, 0, 1) + suffix_field, 0, 1) + ) + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal + Expressing integers in ordinal way for 1-39 and cardinal for 40+ due to Korean grammar. + e.g. 스물세번째 -> ordinal {integer: "23", 23번째} + e.g. 사십오번째 -> ordinal but the integer part is written in cardinal(due to korean grammar) + { integer: "45", 45번쨰} + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="ordinal", kind="classify") + + cardinals = cardinal.just_cardinals + man_as_10000 = pynini.cross("만", "10000") + ordinals_suffix = pynini.accep("번째") # Korean ordinal's morphosyntactic feature + + graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")) # 1-9 in ordinals + cardinal_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) # 1-9 in cardinals + + graph_tens_prefix = pynini.cross("열", "1") # First digit for tens + graph_twenties_prefix = pynini.cross("스물", "2") # First digit for twenties + graph_thirties_prefix = pynini.cross("서른", "3") # First digit for thirties + + # Below exclude regular 1 in ordinal and replace with a special 1. Like "first" in English + # The special 1 is a unique ordinal case for Korean and does not repeat for 11, 21, 31 + graph_one = pynini.cross("한", "1") + single_digits = pynini.project(graph_digit, "input").optimize() + graph_one_acceptor = pynini.project(graph_one, "input").optimize() + two_to_nine = pynini.difference(single_digits, graph_one_acceptor).optimize() + graph_two_to_nine = two_to_nine @ graph_digit + graph_first = pynini.cross("첫", "1") + graph_single = graph_two_to_nine | graph_first + + graph_ten = pynini.cross("열", "10") + graph_tens = graph_ten | graph_tens_prefix + graph_digit + + graph_twenty = pynini.cross("스무", "20") + graph_twenties = graph_twenty | graph_twenties_prefix + graph_digit + + graph_thirty = pynini.cross("서른", "30") + graph_thirties = graph_thirty | graph_thirties_prefix + graph_digit + + ordinals = pynini.union( + graph_single, graph_tens, graph_twenties, graph_thirties # 1-9 # 10-19 # 20-29 # 30-39 + ).optimize() + + cardinal_10_to_19 = pynini.cross("십", "10") | (pynini.accep("십") + cardinal_digit) + + cardinal_20_to_29 = pynini.cross("이십", "20") | (pynini.accep("이십") + cardinal_digit) + + cardinal_30_to_39 = pynini.cross("삼십", "30") | (pynini.accep("삼십") + cardinal_digit) + + # FST that include 1-39 in cardinal expression + cardinal_below_40 = pynini.union( + cardinal_digit, cardinal_10_to_19, cardinal_20_to_29, cardinal_30_to_39 + ).optimize() + + # Input includes all cardinal expressions + cardinals_acceptor = pynini.project(cardinals, "input").optimize() + # Input includes cardinal expression from 1 to 39 + cardinals_exception = pynini.project(cardinal_below_40, "input").optimize() + + # All cardinal values except 1 to 39 cardinal values + cardinal_over_40 = pynini.difference(cardinals_acceptor, cardinals_exception).optimize() + cardinal_ordinal_suffix = cardinal_over_40 @ cardinals + + # 1 to 39 in ordinal, everything else cardinal + ordinal_final = pynini.union(ordinals, cardinal_ordinal_suffix, man_as_10000) + + ordinal_graph = ( + pynutil.insert("integer: \"") + ((ordinal_final + delete_space + ordinals_suffix)) + pynutil.insert("\"") + ) + + # Adding various counter suffix for ordinal + # For counting, Korean does not use the speical "첫" for 1. Instead the regular "한" + counters = pynini.union(graph_digit, graph_tens, graph_twenties, graph_thirties).optimize() + + counter_final = get_counter(counters) | get_counter(cardinal_ordinal_suffix) | get_counter(man_as_10000) + + counter_graph = pynutil.insert("integer: \"") + counter_final + pynutil.insert("\"") + + final_graph = ordinal_graph | counter_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/telephone.py new file mode 100644 index 000000000..381a1188a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/telephone.py @@ -0,0 +1,57 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying a generic 3-4-4 telephone number. + e.g. 공일공에 일이삼사에 오육칠팔 -> telephone { number_part: "010-1234-5678" } + + """ + + def __init__(self): + super().__init__(name="telephone", kind="classify") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_zero_alt = pynini.cross("공", "0") + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + + digit = graph_digit | graph_zero | graph_zero_alt + + separator = pynini.cross(pynini.union(NEMO_SPACE, "에"), "-") + + digit2 = digit**2 + digit3 = digit**3 + digit4 = digit**4 + + optional_separator = pynini.closure(separator, 0, 1) + + phone_number_graph = ( + pynutil.insert('number_part: "') + + pynini.union(digit2, digit3) + + optional_separator + + pynini.union(digit3, digit4) + + optional_separator + + digit4 + + pynutil.insert('"') + ) + + graph = phone_number_graph + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/time.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/time.py new file mode 100644 index 000000000..30080639b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/time.py @@ -0,0 +1,101 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst, delete_space +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time + e.g. 열두시 삼십분 -> time { hours: "12" minutes: "30" } + e.g. 12분전 -> time { minutes: "12" suffix: "전" } + e.g. 새벽 두시 -> time { hours: "2" suffix: "새벽" } + e.g. 두시반 -> time { hours: "2" minutes: "30" } + e.g. 오후 두시반 -> time { prefix: "오후" hours: "2" minutes: "30" } + """ + + def __init__(self): + super().__init__(name="time", kind="classify") + + # 1-9 in cardinals for minutes and seconds + cardinal_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + cardinal_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + graph_tens_prefix = pynini.string_file(get_abs_path("data/time/ten_prefix.tsv")) + + # Graphing 10-19 + graph_ten = pynini.union(pynini.cross("십", "10"), pynini.cross("십", "1") + cardinal_digit).optimize() + # Graphing 20-59 + graph_tens = (graph_tens_prefix + pynini.cross("십", "0")) | ( + graph_tens_prefix + pynini.cross("십", "") + cardinal_digit + ) + + graph_0_to_59 = pynini.union(cardinal_zero, cardinal_digit, graph_ten, graph_tens).optimize() + + # 1-12 for hours + graph_hours = pynini.string_file(get_abs_path("data/time/time_hours.tsv")) + + # Adding space if there are one + spacing = pynini.closure(pynini.accep(NEMO_SPACE), 0, 1) + + hour_suffix = pynini.cross("시", "") + minute_suffix = pynini.cross("분", "") + second_suffix = pynini.cross("초", "") + + hour_component = pynutil.insert("hours: \"") + (graph_hours + spacing + hour_suffix) + pynutil.insert("\"") + + # half minute only allowed after hours: "두시반" / "두시 반" + half_minute_component = pynutil.insert('minutes: "30"') + spacing + pynini.cross("반", "") + + minute_component = ( + pynutil.insert("minutes: \"") + (graph_0_to_59 + spacing + minute_suffix) + pynutil.insert("\"") + ) + + second_component = ( + pynutil.insert("seconds: \"") + (graph_0_to_59 + spacing + second_suffix) + pynutil.insert("\"") + ) + + hm_opt = pynini.closure(delete_space + minute_component, 0, 1) + hs_opt = pynini.closure(delete_space + second_component, 0, 1) + + h_half = hour_component + delete_space + half_minute_component + hs_opt + hms = hour_component + hm_opt + hs_opt + ms = minute_component + pynini.closure(delete_space + second_component, 0, 1) + s_only = second_component + + graph_regular = pynini.union(h_half, hms, ms, s_only).optimize() + + # 오전 = AM, 오후 = PM + ampm_words = pynini.union("오전", "오후", "새벽", "아침") + ampm_tag = pynutil.insert('suffix: "') + ampm_words + pynutil.insert('"') + + # 전 = before, 후 = after + suffix_words = pynini.accep("전") | pynini.accep("후") + suffix_tag = pynutil.insert("suffix: \"") + suffix_words + pynutil.insert("\"") + + time_graph = ( + pynini.closure(delete_space + ampm_tag, 0, 1) + + graph_regular + + pynini.closure(delete_space + suffix_tag, 0, 1) + ) + + # Adding cardinal graph to prevent processing out of range numbers + final_graph = time_graph + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..8e5d39c85 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py @@ -0,0 +1,127 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.ko.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str = INPUT_LOWER_CASED, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ko_itn_{input_case}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + fraction = FractionFst(cardinal, decimal) + fraction_graph = fraction.fst + + time = TimeFst() + time_graph = time.fst + + date = DateFst(cardinal) + date_graph = date.fst + + money = MoneyFst(cardinal, decimal) + money_graph = money.fst + + telephone = TelephoneFst() + telephone_graph = telephone.fst + + measure = MeasureFst(cardinal) + measure_graph = measure.fst + + word_graph = WordFst().fst + whitelist_graph = WhiteListFst().fst + + classify = ( + pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.0) + | pynutil.add_weight(time_graph, 1.0) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + | pynutil.add_weight(whitelist_graph, 1.01) + ) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + space = NEMO_WHITE_SPACE @ delete_extra_space + space_opt = pynini.closure(space, 0, 1) + + graph = delete_space + token + pynini.closure(space_opt + token) + delete_space + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/whitelist.py new file mode 100644 index 000000000..fd443c287 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/whitelist.py @@ -0,0 +1,36 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ko.utils import get_abs_path + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + """ + + def __init__(self): + super().__init__(name="whitelist", kind="classify") + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + graph = (pynutil.insert('name: "')) + (whitelist) + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/taggers/word.py b/nemo_text_processing/inverse_text_normalization/ko/taggers/word.py new file mode 100644 index 000000000..0e4dbb93c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/taggers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. sleep -> tokens { name: "sleep" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/utils.py b/nemo_text_processing/inverse_text_normalization/ko/utils.py new file mode 100644 index 000000000..d198c3835 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/utils.py @@ -0,0 +1,20 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py new file mode 100644 index 000000000..fb9a76d8e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py @@ -0,0 +1,46 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. cardinal { negative: "-" integer: "23" } -> -23 + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + negative_sign = ( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.accep("-") + + pynutil.delete("\"") + ) + + optional_sign_output = pynini.closure(negative_sign + delete_space, 0, 1) + + digits_from_tag = pynini.closure(NEMO_NOT_QUOTE, 1) + integer_cardinal = ( + pynutil.delete("integer:") + delete_space + pynutil.delete("\"") + digits_from_tag + pynutil.delete("\"") + ) + + graph = integer_cardinal + final_graph = optional_sign_output + graph + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/date.py new file mode 100644 index 000000000..88ed973df --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/date.py @@ -0,0 +1,50 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, + e.g. 이천십이년 일월 오일 -> date { year: "2012" month: "1" day: "5" } + e.g. 오월 -> date { month: "5" } + e.g. 칠일 -> date { day: "7" } + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + + year_component = ( + pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.insert("년") + pynutil.delete("\"") + ) + month_component = ( + pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.insert("월") + pynutil.delete("\"") + ) + day_component = ( + pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.insert("일") + pynutil.delete("\"") + ) + + graph = ( + pynini.closure(pynutil.delete(NEMO_SPACE) + year_component, 0, 1) + + pynini.closure(pynutil.delete(NEMO_SPACE) + month_component, 0, 1) + + pynini.closure(pynutil.delete(NEMO_SPACE) + day_component, 0, 1) + ) + + final_graph = self.delete_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py new file mode 100644 index 000000000..65f225f45 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py @@ -0,0 +1,52 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal + e.g. decimal { integer_part: "1" fractional_part: "5" } -> 1.5 + e.g. decimal { integer_part: "1" fractional_part: "5" quantity: "만" } -> 1.5만 + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + + decimal_point = pynutil.insert(".") + integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + fractional_part = pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + quantity_part = pynutil.delete("quantity: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_decimal = integer_part + decimal_point + pynutil.delete(" ") + fractional_part + graph_decimal_larger = ( + integer_part + decimal_point + pynutil.delete(" ") + fractional_part + pynutil.delete(" ") + quantity_part + ) + + graph_sign = pynutil.delete("negative: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph = ( + graph_decimal + | graph_decimal_larger + | (graph_sign + pynutil.delete(" ") + graph_decimal) + | (graph_sign + pynutil.delete(" ") + graph_decimal_larger) + ) + + final_graph = self.delete_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py new file mode 100644 index 000000000..c079bbbca --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py @@ -0,0 +1,76 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( + NEMO_NON_BREAKING_SPACE, + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, +) + + +class FractionFst(GraphFst): + def __init__(self): + """ + Fitite state transducer for classifying fractions + e.g., + fraction { denominator: "사" numerator: "삼" } -> 3/4 + fraction { integer_part: "일" denominator: "사" numerator: "삼" } -> 1 3/4 + fraction { denominator: "루트삼" numerator: "일" } -> 1/√3 + fraction { denominator: "일점육오" numerator: "오십" } -> 50/1.65 + fraction { denominator: "이루트육" numerator: "삼" } -> 3/2√6 + """ + super().__init__(name="fraction", kind="verbalize") + + sign_component = pynutil.delete("negative: \"") + pynini.closure("-", 1) + pynutil.delete("\"") + + mixed_number_component = ( + pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + denominator_component = ( + pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + numerator_component = ( + pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + suffix_component = ( + pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1) + + pynutil.delete('suffix: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + optional_suffix = pynini.closure(pynutil.insert(" ") + suffix_component, 0, 1) + + regular_graph = ( + pynini.closure((sign_component + pynutil.delete(NEMO_SPACE)), 0, 1) + + pynini.closure( + mixed_number_component + pynutil.delete(NEMO_SPACE) + pynutil.insert(NEMO_NON_BREAKING_SPACE) + ) + + numerator_component + + pynutil.delete(NEMO_SPACE) + + pynutil.insert("/") + + denominator_component + + optional_suffix + ) + + final_graph = self.delete_tokens(regular_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/measure.py new file mode 100644 index 000000000..cedf4703d --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/measure.py @@ -0,0 +1,88 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. 십이 킬로그램 -> measure { cardinal { integer: "12" } units: "kg" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self): + super().__init__(name="measure", kind="verbalize") + + measurement = pynini.closure(NEMO_NOT_QUOTE, 1) + + optional_sign = pynini.closure(pynutil.delete('negative: "true"') + delete_space + pynutil.insert("-"), 0, 1) + + unit = pynutil.delete('units: "') + measurement + pynutil.delete('"') + + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + pynutil.delete('integer: "') + + measurement + + pynutil.delete('"') + + delete_space + + pynutil.delete("}") + ) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + pynutil.delete('integer_part: "') + + measurement + + pynutil.delete('"') + + delete_space + + pynutil.delete('fractional_part: "') + + pynutil.insert(".") + + measurement + + pynutil.delete('"') + + delete_space + + pynutil.delete("}") + ) + + graph_fraction = ( + pynutil.delete("fraction {") + + delete_space + + optional_sign + + delete_space + + pynutil.delete('numerator: "') + + measurement + + pynutil.delete('"') + + delete_space + + pynutil.delete('denominator: "') + + pynutil.insert("/") + + measurement + + pynutil.delete('"') + + delete_space + + pynutil.delete("}") + ) + + graph = (graph_cardinal | graph_decimal | graph_fraction) + delete_space + pynutil.insert(" ") + unit + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/money.py new file mode 100644 index 000000000..45e4c7e2c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/money.py @@ -0,0 +1,57 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( + NEMO_CHAR, + NEMO_SPACE, + GraphFst, + delete_space, +) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money + e.g. 오만 삼천원 -> money { integer_part: "53000" currency: "원" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self): + super().__init__(name="money", kind="verbalize") + integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + + pynutil.delete('"') + ) + + unit = ( + pynutil.delete("currency:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + + pynutil.delete('"') + ) + + optional_space = pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1).optimize() + + graph = unit + optional_space + integer + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py new file mode 100644 index 000000000..0fedb457b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py @@ -0,0 +1,47 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 스물세번째 -> ordinal {integer: "23", 23번째} + e.g. 사십오번째 -> ordinal but the integer part is written in cardinal(due to korean grammar) + { integer: "45", 45번쨰} + """ + + def __init__(self): + super().__init__(name="ordinal", kind="verbalize") + + integer_component = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + counter_component = pynutil.delete("counter: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + suffix_component = pynutil.delete('suffix: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + graph_with_counter = ( + integer_component + + delete_space + + counter_component + + pynini.closure(delete_space + suffix_component, 0, 1) + ) + + ordinal_verbalizer = pynini.union(graph_with_counter, integer_component) + + final_graph = self.delete_tokens(ordinal_verbalizer) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py new file mode 100644 index 000000000..96794c610 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py @@ -0,0 +1,33 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying a generic 3-4-4 telephone number. + e.g. 공일공에 일이삼사에 오육칠팔 -> telephone { number: "010-1234-5678" } + + """ + + def __init__(self): + super().__init__(name="telephone", kind="verbalize") + + number_part = pynutil.delete('number_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + delete_tokens = self.delete_tokens(number_part) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/time.py new file mode 100644 index 000000000..4b63ade99 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/time.py @@ -0,0 +1,88 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time + e.g. 열두시 삼십분 -> time { hours: "12" minutes: "30" } + e.g. 12분전 -> time { minutes: "12" suffix: "전" } + e.g. 새벽 두시 -> time { hours: "2" suffix: "새벽" } + e.g. 두시반 -> time { hours: "2" minutes: "30" } + e.g. 오후 두시반 -> time { prefix: "오후" hours: "2" minutes: "30" } + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + + hours_component = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + minutes_component = pynutil.delete("minutes: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + seconds_component = pynutil.delete("seconds: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + suffix_component = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + prefix_component = pynutil.delete("prefix: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + # Add a leading zero to single-digit minutes/seconds + single_digit = NEMO_DIGIT + leading_zero = pynutil.insert("0") + single_digit + add_leading_zero = pynini.union(single_digit @ leading_zero, pynini.closure(NEMO_DIGIT, 2)) + + minutes = minutes_component @ add_leading_zero + seconds = seconds_component @ add_leading_zero + + # Defining all the possible combinations + path_h = hours_component + pynutil.insert(":00") + path_m = minutes + path_s = seconds + + path_hm = hours_component + delete_space + pynutil.insert(":") + minutes + path_hs = ( + hours_component + + delete_space + + pynutil.insert(":") + + pynutil.insert("00") + + delete_space + + pynutil.insert(":") + + seconds + ) + path_ms = minutes + delete_space + pynutil.insert(":") + seconds + + path_hms = ( + hours_component + + delete_space + + pynutil.insert(":") + + minutes + + delete_space + + pynutil.insert(":") + + seconds + ) + + time_graph = pynini.union(path_h, path_m, path_s, path_hm, path_hs, path_ms, path_hms) + + # Adding prefix and suffix space + optional_prefix_out = pynini.closure(delete_space + prefix_component, 0, 1) + optional_suffix_out = pynini.closure(delete_space + pynutil.insert(NEMO_SPACE) + suffix_component, 0, 1) + + final_graph = optional_prefix_out + time_graph + optional_suffix_out + self.fst = self.delete_tokens(delete_space + final_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py new file mode 100644 index 000000000..17e709555 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py @@ -0,0 +1,85 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.fraction import FractionFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.measure import MeasureFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.money import MoneyFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.telephone import TelephoneFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst() + ordinal_graph = ordinal.fst + + decimal = DecimalFst() + decimal_graph = decimal.fst + + fraction = FractionFst() + fraction_graph = fraction.fst + + time = TimeFst() + time_graph = time.fst + + date = DateFst() + date_graph = date.fst + + money = MoneyFst() + money_graph = money.fst + + telephone = TelephoneFst() + telephone_graph = telephone.fst + + measure = MeasureFst() + measure_graph = measure.fst + + word = WordFst() + word_graph = word.fst + + whitelist_graph = WhiteListFst().fst + + graph = pynini.union( + cardinal_graph, + ordinal_graph, + decimal_graph, + fraction_graph, + time_graph, + date_graph, + money_graph, + telephone_graph, + measure_graph, + word_graph, + whitelist_graph, + ) + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py new file mode 100644 index 000000000..6bcca5fb8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py @@ -0,0 +1,45 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py new file mode 100644 index 000000000..395423017 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py @@ -0,0 +1,32 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WhiteListFst(GraphFst): + ''' + tokens { whitelist: "ATM" } -> A T M + ''' + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + + whitelist = pynutil.delete("name: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + graph = whitelist + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ko/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/word.py new file mode 100644 index 000000000..226b41e08 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/ko/verbalizers/word.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.ko.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + GraphFst, + delete_space, +) + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens + e.g. tokens { name: "sleep" } -> sleep + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/__init__.py b/nemo_text_processing/inverse_text_normalization/mr/__init__.py new file mode 100644 index 000000000..9f70e3146 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.mr.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/__init__.py b/nemo_text_processing/inverse_text_normalization/mr/data/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/date/__init__.py b/nemo_text_processing/inverse_text_normalization/mr/data/date/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/date/dates.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/date/dates.tsv new file mode 100644 index 000000000..b77c7dec4 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/date/dates.tsv @@ -0,0 +1,31 @@ +१ एक +२ दोन +३ तीन +४ चार +५ पाच +६ सहा +७ सात +८ आठ +९ नऊ +१० दहा +११ अकरा +१२ बारा +१३ तेरा +१४ चौदा +१५ पंधरा +१६ सोळा +१७ सतरा +१८ अठरा +१९ एकोणीस +२० वीस +२१ एकवीस +२२ बावीस +२३ तेवीस +२४ चोवीस +२५ पंचवीस +२६ सव्वीस +२७ सत्तावीस +२८ अठ्ठावीस +२९ एकोणतीस +३० तीस +३१ एकतीस \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/date/months.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/date/months.tsv new file mode 100644 index 000000000..f39b90d2c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/date/months.tsv @@ -0,0 +1,12 @@ +जानेवारी +फेब्रुवारी +मार्च +एप्रिल +मे +जून +जुलै +ऑगस्ट +सप्टेंबर +ऑक्टोबर +नोव्हेंबर +डिसेंबर \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/date/prefixes.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/date/prefixes.tsv new file mode 100644 index 000000000..dce4c03b5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/date/prefixes.tsv @@ -0,0 +1,2 @@ +इसवी सन पूर्व इ.स.पू. +इसवी सन इ.स. \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/numbers/digits.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/digits.tsv new file mode 100644 index 000000000..f737ade2c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/digits.tsv @@ -0,0 +1,9 @@ +१ एक +२ दोन +३ तीन +४ चार +५ पाच +६ सहा +७ सात +८ आठ +९ नऊ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/numbers/hundred.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/hundred.tsv new file mode 100644 index 000000000..5b05d55e9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/hundred.tsv @@ -0,0 +1 @@ +१०० शंभर \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/numbers/tens.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/tens.tsv new file mode 100644 index 000000000..81107a31a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/tens.tsv @@ -0,0 +1,90 @@ +१० दहा +११ अकरा +१२ बारा +१३ तेरा +१४ चौदा +१५ पंधरा +१६ सोळा +१७ सतरा +१८ अठरा +१९ एकोणीस +२० वीस +२१ एकवीस +२२ बावीस +२३ तेवीस +२४ चोवीस +२५ पंचवीस +२६ सव्वीस +२७ सत्तावीस +२८ अठ्ठावीस +२९ एकोणतीस +३० तीस +३१ एकतीस +३२ बत्तीस +३३ तेहतीस +३४ चौतीस +३५ पस्तीस +३६ छत्तीस +३७ सदतीस +३८ अडतीस +३९ एकोणचाळीस +४० चाळीस +४१ एकेचाळीस +४२ बेचाळीस +४३ त्रेचाळीस +४४ चव्वेचाळीस +४५ पंचेचाळीस +४६ सेहचाळीस +४७ सत्तेचाळीस +४८ अठ्ठेचाळीस +४९ एकोणपन्नास +५० पन्नास +५१ एकावन्न +५२ बावन्न +५३ त्रेपन्न +५४ चौपन्न +५५ पंचावन्न +५६ छप्पन +५७ सत्तावन्न +५८ अठ्ठावन्न +५९ एकोणसाठ +६० साठ +६१ एकसष्ट +६२ बासष्ट +६३ त्रेसष्ट +६४ चौसष्ट +६५ पासष्ट +६६ सहासष्ठ +६७ सदुसष्ट +६८ अडुसष्ठ +६९ एकोणसत्तर +७० सत्तर +७१ एकाहत्तर +७२ बाहत्तर +७३ त्र्याहत्तर +७४ चौऱ्याहत्तर +७५ पंचाहत्तर +७६ शहात्तर +७७ सत्याहत्तर +७८ अठ्ठ्यात्तर +७९ एकोणऐंशी +८० ऐंशी +८१ एक्याऐंशी +८२ ब्याऐंशी +८३ त्र्याऐंशी +८४ चौऱ्याऐंशी +८५ पंच्याऐंशी +८६ सह्यांशी +८७ सत्याऐंशी +८८ अठ्ठ्याऐंशी +८९ एकोणनव्वद +९० नव्वद +९१ एक्याण्णव +९२ ब्याण्णव +९३ त्र्याण्णव +९४ चौऱ्याण्णव +९५ पंच्याण्णव +९६ शह्याण्णवx +९७ सत्त्याण्णव +९८ अठ्ठ्याण्णव +९९ नव्व्याण्णव \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/numbers/thousands.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/thousands.tsv new file mode 100644 index 000000000..f9f09996c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/thousands.tsv @@ -0,0 +1,4 @@ +हजार +लाख +कोटी +अब्ज \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/zero.tsv new file mode 100644 index 000000000..727ccdf16 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/numbers/zero.tsv @@ -0,0 +1 @@ +० शून्य \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/time/__init__.py b/nemo_text_processing/inverse_text_normalization/mr/data/time/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/time/hours.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/time/hours.tsv new file mode 100644 index 000000000..b89645ec9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/time/hours.tsv @@ -0,0 +1,12 @@ +एक १ +दोन २ +तीन ३ +चार ४ +पाच ५ +सहा ६ +सात ७ +आठ ८ +नऊ ९ +दहा १० +अकरा ११ +बारा १२ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/time/hours_to.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/time/hours_to.tsv new file mode 100644 index 000000000..34d69d331 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/time/hours_to.tsv @@ -0,0 +1,12 @@ +एक १२ +दोन १ +तीन २ +चार ३ +पाच ४ +सहा ५ +सात ६ +आठ ७ +नऊ ८ +दहा ९ +अकरा १० +बारा ११ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/time/minutes.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/time/minutes.tsv new file mode 100644 index 000000000..e9ae86c0b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/time/minutes.tsv @@ -0,0 +1,60 @@ +एक १ +दोन २ +तीन ३ +चार ४ +पाच ५ +सहा ६ +सात ७ +आठ ८ +नऊ ९ +दहा १० +अकरा ११ +बारा १२ +तेरा १३ +चौदा १४ +पंधरा १५ +सोळा १६ +सतरा १७ +अठरा १८ +एकोणीस १९ +वीस २० +एकवीस २१ +बावीस २२ +तेवीस २३ +चोवीस २४ +पंचवीस २५ +सव्वीस २६ +सत्तावीस २७ +अठ्ठावीस २८ +एकोणतीस २९ +तीस ३० +एकतीस ३१ +बत्तीस ३२ +तेहतीस ३३ +चौतीस ३४ +पस्तीस ३५ +छत्तीस ३६ +सदतीस ३७ +अडतीस ३८ +एकोणचाळीस ३९ +चाळीस ४० +एकेचाळीस ४१ +बेचाळीस ४२ +त्रेचाळीस ४३ +चव्वेचाळीस ४४ +पंचेचाळीस ४५ +सेहेचाळीस ४६ +सत्तेचाळीस ४७ +अठ्ठेचाळीस ४८ +एकोणपन्नास ४९ +पन्नास ५० +एकावन्न ५१ +बावन्न ५२ +त्रेपन्न ५३ +चौपन्न ५४ +पंचावन्न ५५ +छप्पन ५६ +सत्तावन्न ५७ +अठ्ठावन्न ५८ +एकोणसाठ ५९ +साठ ६० \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/data/time/minutes_to.tsv b/nemo_text_processing/inverse_text_normalization/mr/data/time/minutes_to.tsv new file mode 100644 index 000000000..01c63cc76 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/data/time/minutes_to.tsv @@ -0,0 +1,59 @@ +एक ५९ +दोन ५८ +तीन ५७ +चार ५६ +पाच ५५ +सहा ५४ +सात ५३ +आठ ५२ +नऊ ५१ +दहा ५० +अकरा ४९ +बारा ४८ +तेरा ४७ +चौदा ४६ +पंधरा ४५ +सोळा ४४ +सतरा ४३ +अठरा ४२ +एकोणीस ४१ +वीस ४० +एकवीस ३९ +बावीस ३८ +तेवीस ३७ +चोवीस ३६ +पंचवीस ३५ +सव्वीस ३४ +सत्तावीस ३३ +अठ्ठावीस ३२ +एकोणतीस ३२ +तीस ३० +एकतीस २९ +बत्तीस २८ +तेहतीस २७ +चौतीस २६ +पस्तीस २५ +छत्तीस २४ +सदतीस २३ +अडतीस २२ +एकोणचाळीस २१ +चाळीस २० +एकेचाळीस १९ +बेचाळीस १८ +त्रेचाळीस १७ +चव्वेचाळीस १६ +पंचेचाळीस १५ +सेहचाळीस १४ +सत्तेचाळीस १३ +अठ्ठेचाळीस १२ +एकोणपन्नास ११ +पन्नास १० +एकावन्न ९ +बावन्न ८ +त्रेपन्न ७ +चौपन्न ६ +पंचावन्न ५ +छप्पन ४ +सत्तावन्न ३ +अठ्ठावन्न २ +एकोणसाठ १ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/mr/graph_utils.py b/nemo_text_processing/inverse_text_normalization/mr/graph_utils.py new file mode 100644 index 000000000..eaac1ba33 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/graph_utils.py @@ -0,0 +1,194 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import pynutil, utf8 + +from nemo_text_processing.inverse_text_normalization.mr.utils import load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR +NEMO_MARATHI_DIGITS = ( + "\u0966" + "\u0967" + "\u0968" + "\u0969" + "\u096a" + "\u096b" + "\u096c" + "\u096d" + "\u096e" + "\u096f" +) +NEMO_DIGIT = pynini.union(*NEMO_MARATHI_DIGITS).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_CHAR, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("उणे").optimize() + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/cardinal.py new file mode 100644 index 000000000..8aa218a9a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/cardinal.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import ( + MINUS, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.mr.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. तेहतीस -> cardinal { integer: "३३" } + """ + + def __init__(self): + super().__init__(name="cardinal", kind="classify") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + graph_digits = pynini.string_file(get_abs_path("data/numbers/digits.tsv")).invert() + graph_tens = pynini.string_file(get_abs_path("data/numbers/tens.tsv")).invert() + graph_hundred_unique = pynini.string_file(get_abs_path("data/numbers/hundred.tsv")).invert() + + graph_hundred = pynini.cross("शे", "") + + graph_hundred_component = pynini.union(graph_digits + graph_hundred, pynutil.insert("०")) + graph_hundred_component += delete_space + graph_hundred_component += pynini.union(pynutil.insert("००"), graph_tens, pynutil.insert("०") + graph_digits) + + graph_hundred_component_at_least_one_non_zero_digit = graph_hundred_component @ ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "०") + pynini.closure(NEMO_DIGIT) + ) + self.graph_hundred_component_at_least_one_non_zero_digit = graph_hundred_component_at_least_one_non_zero_digit + + # eleven hundred -> 1100 etc form + graph_hundred_as_thousand = graph_tens + graph_hundred + graph_hundred_as_thousand += delete_space + pynini.union( + pynutil.insert("००"), graph_tens, pynutil.insert("०") + graph_digits + ) + + graph_hundreds = graph_hundred_component | graph_hundred_as_thousand + + graph_two_digit_component = pynini.union(pynutil.insert("००"), graph_tens, pynutil.insert("०") + graph_digits) + + graph_two_digit_component_at_least_one_non_zero_digit = graph_two_digit_component @ ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "०") + pynini.closure(NEMO_DIGIT) + ) + self.graph_two_digit_component_at_least_one_non_zero_digit = ( + graph_two_digit_component_at_least_one_non_zero_digit + ) + + graph_thousands = pynini.union( + graph_two_digit_component_at_least_one_non_zero_digit + delete_space + pynutil.delete("हजार"), + pynutil.insert("००", weight=0.1), + ) + + graph_lakhs = pynini.union( + graph_two_digit_component_at_least_one_non_zero_digit + delete_space + pynutil.delete("लाख"), + pynutil.insert("००", weight=0.1), + ) + + graph_crores = pynini.union( + graph_two_digit_component_at_least_one_non_zero_digit + delete_space + pynutil.delete("कोटी"), + pynutil.insert("००", weight=0.1), + ) + + graph_arabs = pynini.union( + graph_two_digit_component_at_least_one_non_zero_digit + delete_space + pynutil.delete("अब्ज"), + pynutil.insert("००", weight=0.1), + ) + + graph_higher_powers = ( + graph_arabs + delete_space + graph_crores + delete_space + graph_lakhs + delete_space + graph_thousands + ) + + graph = pynini.union( + graph_higher_powers + delete_space + graph_hundreds, + graph_hundred_unique, + graph_zero, + ) + + graph = graph @ pynini.union( + pynutil.delete(pynini.closure("०")) + pynini.difference(NEMO_DIGIT, "०") + pynini.closure(NEMO_DIGIT), "०" + ) + graph = graph.optimize() + + self.graph = (pynini.project(graph, "input")) @ graph + + optional_minus_graph = pynini.closure( + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"-\"") + NEMO_SPACE, 0, 1 + ) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/date.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/date.py new file mode 100644 index 000000000..15a75affc --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/date.py @@ -0,0 +1,65 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.mr.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite State Transducer for classifying dates + e.g. दहा जानेवारी दोन हजार -> date { day: "१०" month: "जानेवारी" year: "२०००" preserve_order: true } + e.g. इसवी सन दोन हजार बावीस -> date { text: "इ.स." year: "२०२२" preserve_order: true } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name='date', kind="classify") + months = pynini.string_file(get_abs_path("data/date/months.tsv")) + dates = pynini.string_file(get_abs_path("data/date/dates.tsv")).invert() + prefixes = pynini.string_file(get_abs_path("data/date/prefixes.tsv")) + + YEAR_WEIGHT = 0.001 + month_graph = pynutil.insert("month: \"") + months + pynutil.insert("\" ") + day_graph = pynutil.insert("day: \"") + dates + pynutil.insert("\" ") + year_graph = cardinal.graph + graph_year = ( + delete_extra_space + + pynutil.insert("year: \"") + + pynutil.add_weight(year_graph, -YEAR_WEIGHT) + + pynutil.insert("\"") + ) + optional_graph_year = pynini.closure( + graph_year, + 0, + 1, + ) + graph_ad_bc = pynutil.insert("text: \"") + prefixes + delete_space + pynutil.insert("\"") + + graph_mdy = month_graph + ( + (delete_extra_space + day_graph) | graph_year | (delete_extra_space + day_graph + graph_year) + ) + graph_dmy = day_graph + delete_space + month_graph + optional_graph_year + graph_year_prefix = graph_ad_bc + graph_year + + final_graph = graph_mdy | graph_dmy | graph_year_prefix + final_graph += pynutil.insert(" preserve_order: true") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/decimal.py new file mode 100644 index 000000000..92af8c7c3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/decimal.py @@ -0,0 +1,100 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import ( + MINUS, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + delete_extra_space, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.mr.utils import get_abs_path, load_labels + + +def get_quantity(decimal, cardinal_fst): + numbers = cardinal_fst @ ( + pynutil.delete(pynini.closure("०")) + pynini.difference(NEMO_DIGIT, "०") + pynini.closure(NEMO_DIGIT) + ) + suffix_labels = load_labels(get_abs_path("/data/numbers/thousands.tsv")) + suffix_labels = [x[0] for x in suffix_labels if x[0] != "हजार"] + suffix = pynini.union(*suffix_labels).optimize() + + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + delete_extra_space + + pynutil.insert("quantity: \"") + + suffix + + pynutil.insert("\"") + ) + res |= decimal + delete_extra_space + pynutil.insert("quantity: \"") + (suffix | "हजार") + pynutil.insert("\"") + + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. तेहतीस पूर्णांक तीन -> decimal { integer_part: "३३" fractional_part: "३" } + e.g. उणे तेहतीस पूर्णांक तीन लाख -> decimal { negative: "true" integer_part: "३३" fractional_part: "३" quantity: "लाख" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="decimal", kind="classify") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + graph_digits = pynini.string_file(get_abs_path("data/numbers/digits.tsv")).invert() + decimal_word = pynini.cross("पूर्णांक", "") + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space, + 0, + 1, + ) + graph_integer = ( + pynutil.insert("integer_part: \"") + + pynini.closure(cardinal.graph, 0, 1) + + pynutil.insert("\"") + + NEMO_SPACE + ) + graph_decimal = graph_integer + delete_space + decimal_word + + graph_fractional = ( + pynutil.insert("fractional_part: \"") + + pynini.closure(delete_space + (graph_zero | graph_digits), 1) + + pynutil.insert("\"") + ) + graph_decimal += graph_fractional + + final_graph_without_sign = graph_decimal + final_graph = optional_graph_negative + final_graph_without_sign + + self.final_graph_without_negative = final_graph_without_sign | get_quantity( + final_graph_without_sign, cardinal.graph_hundred_component_at_least_one_non_zero_digit + ) + + quantity_graph = get_quantity( + final_graph_without_sign, cardinal.graph_hundred_component_at_least_one_non_zero_digit + ) + final_graph |= optional_graph_negative + quantity_graph + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/punctuation.py new file mode 100644 index 000000000..97b5257a0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/punctuation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/time.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/time.py new file mode 100644 index 000000000..b6e1080da --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/time.py @@ -0,0 +1,117 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import GraphFst, delete_space +from nemo_text_processing.inverse_text_normalization.mr.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time + e.g. साडे चार -> time { hours: "४" minutes: "३०" } + e.g. सव्वा बारा -> time { hours: "१२" minutes: "१५" } + e.g. पावणे दहा -> time { hours: "९" minutes: "४५" } + e.g. अकराला पाच मिनिटे -> time { hours: "१०" minutes: "५५" } + e.g. अकरा वाजून दोन मिनिटे -> time { hours: "११" minutes: "२" } + e.g. अडीच -> time { hours: "२" minutes: "३०" } + """ + + def __init__(self): + super().__init__(name="time", kind="classify") + hours = pynini.string_file(get_abs_path("data/time/hours.tsv")) + minutes = pynini.string_file(get_abs_path("data/time/minutes.tsv")) + hours_to = pynini.string_file(get_abs_path("data/time/hours_to.tsv")) + minutes_to = pynini.string_file(get_abs_path("data/time/minutes_to.tsv")) + + time_word = pynini.cross("वाजून", "") + minutes_word = pynini.cross("मिनिटे", "") | pynini.cross("मिनिट", "") + graph_time_full = ( + pynutil.insert("hours: \"") + + hours + + pynutil.insert("\"") + + delete_space + + time_word + + delete_space + + pynutil.insert(" ") + + pynutil.insert("minutes: \"") + + minutes + + pynutil.insert("\"") + + delete_space + + minutes_word + ) + graph_time_to = ( + pynutil.insert("hours: \"") + + hours_to + + pynutil.insert("\"") + + pynini.cross("ला", "") + + delete_space + + pynutil.insert(" ") + + pynutil.insert("minutes: \"") + + minutes_to + + pynutil.insert("\"") + + delete_space + + minutes_word + ) + + # special terms used for 15, 30 and 45 minutes + graph_fifteen = ( + pynini.cross("सव्वा", "") + + delete_space + + pynutil.insert("hours: \"") + + hours + + pynutil.insert("\"") + + pynutil.insert(" ") + + pynutil.insert("minutes: \"") + + pynutil.insert("१५") + + pynutil.insert("\"") + ) + graph_thirty = ( + pynini.cross("साडे", "") + + delete_space + + pynutil.insert("hours: \"") + + hours + + pynutil.insert("\"") + + pynutil.insert(" ") + + pynutil.insert("minutes: \"") + + pynutil.insert("३०") + + pynutil.insert("\"") + ) + graph_fortyfive = ( + pynini.cross("पावणे", "") + + delete_space + + pynutil.insert("hours: \"") + + hours_to + + pynutil.insert("\"") + + pynutil.insert(" ") + + pynutil.insert("minutes: \"") + + pynutil.insert("४५") + + pynutil.insert("\"") + ) + + special_cases = (pynini.cross("दीड", "") + pynutil.insert("hours: \"१\" minutes: \"३०\"")) | ( + pynini.cross("अडीच", "") + pynutil.insert("hours: \"२\" minutes: \"३०\"") + ) + + graph = pynini.union( + graph_time_full, graph_time_to, graph_fifteen, graph_thirty, graph_fortyfive, special_cases + ) + + final_graph = graph + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..0409b0a25 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/tokenize_and_classify.py @@ -0,0 +1,97 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import ( + INPUT_LOWER_CASED, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.inverse_text_normalization.mr.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.mr.taggers.date import DateFst +from nemo_text_processing.inverse_text_normalization.mr.taggers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.mr.taggers.punctuation import PunctuationFst +from nemo_text_processing.inverse_text_normalization.mr.taggers.time import TimeFst +from nemo_text_processing.inverse_text_normalization.mr.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + input_case: str = INPUT_LOWER_CASED, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"mr_itn_{input_case}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + decimal_graph = DecimalFst(cardinal).fst + time_graph = TimeFst().fst + date_graph = DateFst(cardinal).fst + + word_graph = WordFst().fst + punct_graph = PunctuationFst().fst + classify = ( + pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(date_graph, 1.09) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/mr/taggers/word.py b/nemo_text_processing/inverse_text_normalization/mr/taggers/word.py new file mode 100644 index 000000000..2a062e5a3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/taggers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. चालणे -> tokens { name: "चालणे" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/utils.py b/nemo_text_processing/inverse_text_normalization/mr/utils.py new file mode 100644 index 000000000..1f1349115 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/utils.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/cardinal.py new file mode 100644 index 000000000..5ca2f361c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/cardinal.py @@ -0,0 +1,53 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import NEMO_DIGIT, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. cardinal { negative: "-" integer: "३३६२००" } : -३३६२०० + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.accep("-") + + pynutil.delete("\"") + + delete_space, + 0, + 1, + ) + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) # Accepts at least one digit change nemo digit to whatever is relevant + + pynutil.delete("\"") + + delete_space + ) + # graph = optional_sign + graph # concatenates two properties + graph = optional_sign + graph + delete_tokens = self.delete_tokens(graph) # removes semiotic class tag + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/date.py new file mode 100644 index 000000000..617d55449 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/date.py @@ -0,0 +1,92 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_extra_space, + delete_space, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + e.g. date { month: "जानेवारी" year: "२०००" preserve_order: true } -> जानेवारी २००० + e.g. date { day: "१५" month: "फेब्रुवारी" year: "२०२३" } -> १५ फेब्रुवारी २०२३ + e.g. date { text: "इ.स.पू." year: "१९८५" preserve_order: true } -> इ.स.पू. १९८५ + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + month = ( + pynutil.delete("month:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + day = ( + pynutil.delete("day:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + year = ( + pynutil.delete("year:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + delete_space + + pynutil.delete("\"") + ) + period = ( + pynutil.delete("text:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + graph_fy = period + pynini.closure(delete_extra_space + year, 0, 1) + # month (day) year + graph_mdy = ( + month + pynini.closure(delete_extra_space + day, 0, 1) + pynini.closure(delete_extra_space + year, 0, 1) + ) + + # (day) month year + graph_dmy = ( + pynini.closure(day + delete_extra_space, 0, 1) + month + pynini.closure(delete_extra_space + year, 0, 1) + ) + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + final_graph = (graph_mdy | year | graph_dmy | graph_fy) + delete_space + optional_preserve_order | ( + graph_mdy | year | graph_dmy | graph_fy + ) + delete_space + optional_preserve_order + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/decimal.py new file mode 100644 index 000000000..1976fae24 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/decimal.py @@ -0,0 +1,62 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal + e.g. decimal { integer_part: "३००" fractional_part: "०३१" } -> ३००.०३१ + e.g. decimal { negative: "true" integer_part: "७३" fractional_part: "५" quantity: "लाख" } -> -७३.५ लाख + e.g. decimal { integer_part: "००८" fractional_part: "५०" } -> ८.५० + """ + + def __init__(self): + super().__init__(name="decimal", kind="verbalize") + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-") + delete_space, 0, 1) + integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_integer = pynini.closure(integer + delete_space, 0, 1) + fractional = ( + pynutil.insert(".") + + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_fractional = pynini.closure(fractional + delete_space, 0, 1) + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1) + graph = optional_integer + optional_fractional + optional_quantity + self.numbers = graph + graph = optional_sign + graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/time.py new file mode 100644 index 000000000..15fcf6e45 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/time.py @@ -0,0 +1,55 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import NEMO_DIGIT, GraphFst, delete_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + e.g. time { hours: "४" minutes: "३०" } -> ०४:३० + e.g. time { hours: "११" minutes: "३०" } -> ११:३० + e.g. time { hours: "८" minutes: "१५" } -> ०८:१५ + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("०") + NEMO_DIGIT) + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + minute = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_DIGIT, 1) + + pynutil.delete("\"") + ) + graph = ( + (hour @ add_leading_zero_to_double_digit) + + delete_space + + pynutil.insert(":") + + (minute @ add_leading_zero_to_double_digit) + ) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize.py new file mode 100644 index 000000000..ca729c37b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize.py @@ -0,0 +1,37 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.date import DateFst +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.decimal import DecimalFst +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + cardinal_graph = CardinalFst().fst + decimal_graph = DecimalFst().fst + time_graph = TimeFst().fst + date_graph = DateFst().fst + graph = cardinal_graph | decimal_graph | time_graph | date_graph + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize_final.py new file mode 100644 index 000000000..2b4dccc7e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/verbalize_final.py @@ -0,0 +1,45 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.mr.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "दुपारचे" } tokens { time { hours: "२" minutes: "३०" } } tokens { name: "वाजले" } tokens { name: "आहेत" } -> दुपारचे 0२:३० वाजले आहेत + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/mr/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/word.py new file mode 100644 index 000000000..f3ca79d5a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/mr/verbalizers/word.py @@ -0,0 +1,39 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.mr.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + GraphFst, + delete_space, +) + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain tokens + e.g. tokens { name: "चालणे" } -> चालणे + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py index 5ff16a6e0..59b30ae9e 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/cardinal.py @@ -14,6 +14,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, @@ -24,7 +26,6 @@ GraphFst, delete_space, ) -from pynini.lib import pynutil class CardinalFst(GraphFst): @@ -170,9 +171,9 @@ def __init__(self, use_strict_e=False): ) @ (pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)) graph_hundred_component_prefix_e = graph_hundred_component_prefix_e.optimize() - graph_hundred_component_no_prefix = pynini.union(graph_hundreds + graph_e + graph_ties_component,) @ ( - pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) - ) + graph_hundred_component_no_prefix = pynini.union( + graph_hundreds + graph_e + graph_ties_component, + ) @ (pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT)) graph_hundred_component_no_prefix = graph_hundred_component_no_prefix.optimize() graph_mil_prefix_e = pynini.union( @@ -349,18 +350,13 @@ def __init__(self, use_strict_e=False): self.graph_no_exception = graph # save self.numbers_up_to_thousand for use in DecimalFst - digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT ** 2) | (NEMO_DIGIT ** 3) + digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3) numbers_up_to_thousand = pynini.compose(graph, digits_up_to_thousand).optimize() self.numbers_up_to_thousand = numbers_up_to_thousand # save self.numbers_up_to_million for use in DecimalFst digits_up_to_million = ( - NEMO_DIGIT - | (NEMO_DIGIT ** 2) - | (NEMO_DIGIT ** 3) - | (NEMO_DIGIT ** 4) - | (NEMO_DIGIT ** 5) - | (NEMO_DIGIT ** 6) + NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3) | (NEMO_DIGIT**4) | (NEMO_DIGIT**5) | (NEMO_DIGIT**6) ) numbers_up_to_million = pynini.compose(graph, digits_up_to_million).optimize() self.numbers_up_to_million = numbers_up_to_million diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/date.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/date.py index 4b891ea98..2cd22e5ca 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py index dab779965..24981ed7e 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, @@ -20,7 +22,6 @@ delete_extra_space, delete_space, ) -from pynini.lib import pynutil def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_million: 'pynini.FstLike') -> 'pynini.FstLike': diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py index aa152b116..4f3fe1fc8 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, GraphFst, insert_space -from pynini.lib import pynutil class ElectronicFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py index 7b6f1015a..13b9ffef3 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_SIGMA, @@ -21,7 +23,6 @@ delete_extra_space, delete_space, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/money.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/money.py index cc3639438..5970da8de 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, @@ -23,7 +25,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py index ff7f3fbf0..dae122abc 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst, delete_space -from pynini.lib import pynutil class OrdinalFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py index cb5285452..3c23bad1a 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/punctuation.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class PunctuationFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py index a1ad2d075..d048cc36c 100755 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/telephone.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space, insert_space -from pynini.lib import pynutil class TelephoneFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/time.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/time.py index 1e209ec96..df2a6e69e 100755 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/time.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_space, insert_space -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py index 2ad382b4a..94282af34 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.pt.taggers.date import DateFst from nemo_text_processing.inverse_text_normalization.pt.taggers.decimal import DecimalFst @@ -35,7 +36,7 @@ delete_space, generator_main, ) -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -66,9 +67,9 @@ def __init__( far_file = os.path.join(cache_dir, f"pt_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") cardinal = CardinalFst(use_strict_e=True) cardinal_graph = cardinal.fst @@ -115,4 +116,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py index 5f6491893..3377f39a9 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/taggers/word.py b/nemo_text_processing/inverse_text_normalization/pt/taggers/word.py index 7908397d5..1544fb351 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/pt/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py index 928a259d3..56c82501f 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py index 7c0034099..8faf99407 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py index 58fc76ea6..60a61e0ee 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py index 11b2706a3..203a44682 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class ElectronicFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py index 057ade696..77d62d93f 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py index 54a9b1038..755361bda 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, delete_space, insert_space + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py index fe3454e15..25d8595df 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py index 4dd0d7079..1423f33e8 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/telephone.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class TelephoneFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py index b1a04c673..4bdfe0fca 100755 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py index cc2e65aed..5e7248bac 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/verbalize_final.py @@ -13,10 +13,11 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.pt.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.pt.verbalizers.word import WordFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py index f54aaea65..6d9859c6e 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/whitelist.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WhiteListFst(GraphFst): """ @@ -33,5 +34,5 @@ def __init__(self): + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete("\"") ) - graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py index 4417d8f00..aba72eaf8 100644 --- a/nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/pt/verbalizers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WordFst(GraphFst): """ @@ -27,6 +28,6 @@ def __init__(self): super().__init__(name="word", kind="verbalize") chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/cardinal.py index e812be117..cfb6add51 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/cardinal.py @@ -14,13 +14,14 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space + class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. "тысяча один" -> cardinal { integer: "1 001" } Args: diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/date.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/date.py index 9727626c8..8e262891b 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/date.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class DateFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/decimals.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/decimals.py index 66e8f5f73..aa6c784b3 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/decimals.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/decimals.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst, delete_extra_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst, delete_extra_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/electronic.py index f73c8f1b8..62712c054 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class ElectronicFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/measure.py index ef15a61e2..d083d899b 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/money.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/money.py index 7477350ec..0185fed32 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/money.py @@ -13,9 +13,10 @@ # limitations under the License. -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/ordinal.py index 306054d63..d0d62f193 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/ordinal.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/telephone.py index 7b54ddc7e..61bbbad20 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/telephone.py @@ -13,9 +13,10 @@ # limitations under the License. -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class TelephoneFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/time.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/time.py index 50e12c09a..01e12453c 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/time.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst from nemo_text_processing.text_normalization.ru.verbalizers.time import TimeFst as TNTimeVerbalizer -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py index 00cd1f1ae..75469203d 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.taggers.punctuation import PunctuationFst from nemo_text_processing.inverse_text_normalization.en.taggers.word import WordFst from nemo_text_processing.inverse_text_normalization.ru.taggers.cardinal import CardinalFst @@ -36,7 +37,7 @@ generator_main, ) from nemo_text_processing.text_normalization.ru.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -67,9 +68,9 @@ def __init__( far_file = os.path.join(cache_dir, f"ru_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") tn_classify = TNClassifyFst( input_case='cased', deterministic=False, cache_dir=cache_dir, overwrite_cache=True ) @@ -119,4 +120,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/ru/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/ru/taggers/whitelist.py index 45edf1592..2c44cdac7 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/ru/taggers/whitelist.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py index 12e2451f7..fa76f2fbc 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/date.py index 02cce48da..8ccf4c887 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class DateFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py index cf8cefc5d..62da0ddb5 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py index 158cd9893..0b560b82e 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class ElectronicFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/measure.py index 3cae67bac..5ba92e6b0 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/money.py index af59d33db..9c8a44b00 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py index 31ff9b3c6..3627ede9a 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py index 06c705dc4..47e1c2754 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/telephone.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class TelephoneFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/time.py index edfe7b898..addd809eb 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/time.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class TimeFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py index f409537d2..53a325c15 100644 --- a/nemo_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/ru/verbalizers/verbalize_final.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.inverse_text_normalization.ru.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now """ diff --git a/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/nemo_text_processing/inverse_text_normalization/run_evaluate.py index d2b3b6330..cf9b29fce 100644 --- a/nemo_text_processing/inverse_text_normalization/run_evaluate.py +++ b/nemo_text_processing/inverse_text_normalization/run_evaluate.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ training_data_to_tokens, ) - ''' Runs Evaluation on data in the format of : \t\t<`self` if trivial class or normalized text> like the Google text normalization data https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish @@ -34,8 +33,32 @@ def parse_args(): parser = ArgumentParser() parser.add_argument("--input", help="input file path", type=str) parser.add_argument( - "--lang", help="language", choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'vi'], default="en", type=str + "--lang", + help="language", + choices=[ + "ar", + "de", + "en", + "es", + "es_en", + "fr", + "hi", + "hi_en", + "hy", + "ko", + "mr", + "pt", + "ru", + "sv", + "vi", + "zh", + 'ja', + ], + default="en", + type=str, ) + parser.add_argument("--input_case", choices=["lower_cased", "cased"]) + parser.add_argument("--output_case", choices=["lower_cased", "cased"]) parser.add_argument( "--cat", dest="category", @@ -55,10 +78,15 @@ def parse_args(): if args.lang == 'en': from nemo_text_processing.inverse_text_normalization.en.clean_eval_data import filter_loaded_data file_path = args.input - inverse_normalizer = InverseNormalizer() + inverse_normalizer = InverseNormalizer(lang=args.lang, input_case=args.input_case) print("Loading training data: " + file_path) - training_data = load_files([file_path]) + if args.output_case == "lower_cased": + to_lower = True + elif args.output_case == "cased": + to_lower = False + + training_data = load_files([file_path], to_lower=to_lower) if args.filter: training_data = filter_loaded_data(training_data) @@ -68,6 +96,10 @@ def parse_args(): sentences_un_normalized, sentences_normalized, _ = training_data_to_sentences(training_data) print("- Data: " + str(len(sentences_normalized)) + " sentences") sentences_prediction = inverse_normalizer.inverse_normalize_list(sentences_normalized) + with open('result.log', 'w') as ofp: + for inp, out in zip(sentences_un_normalized, sentences_prediction): + ofp.write(f'{inp==out}; {inp}\t{out}\n') + print("- Denormalized. Evaluating...") sentences_accuracy = evaluate( preds=sentences_prediction, labels=sentences_un_normalized, input=sentences_normalized diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/cardinal.py index 07c1cd9b6..d352284be 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/cardinal.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst + class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals. Numbers below ten are not converted. + Finite state transducer for classifying cardinals. Numbers below ten are not converted. Allows both compound numeral strings or separated by whitespace. e.g. minus tjugoen -> cardinal { negative: "-" integer: "21" } } diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/date.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/date.py index eb10d0753..5d9308958 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/date.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst + class DateFst(GraphFst): """ @@ -30,7 +31,8 @@ class DateFst(GraphFst): """ def __init__( - self, tn_date_tagger: GraphFst, + self, + tn_date_tagger: GraphFst, ): super().__init__(name="date", kind="classify") diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/decimal.py index da3d7e493..97bd36582 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.sv.taggers.decimal import get_quantity -from pynini.lib import pynutil class DecimalFst(GraphFst): @@ -48,7 +49,15 @@ def __init__(self, itn_cardinal_tagger: GraphFst, tn_decimal_tagger: GraphFst): self.final_graph_wo_sign = final_graph_wo_sign self.final_graph_wo_negative = ( - final_graph_wo_sign | get_quantity(final_graph_wo_sign, None, hundreds_no_one, None, False, True,) + final_graph_wo_sign + | get_quantity( + final_graph_wo_sign, + None, + hundreds_no_one, + None, + False, + True, + ) ).optimize() optional_minus_graph = pynini.closure(pynini.cross("minus ", "negative: \"true\" "), 0, 1) diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/electronic.py index 526d45070..484efff78 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/electronic.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class ElectronicFst(GraphFst): """ Finite state transducer for classifying electronic: email addresses, etc. e.g. c d f ett at a b c punkt e d u -> tokens { name: "cdf1.abc.edu" } - + Args: tn_electronic_tagger: TN eletronic tagger tn_electronic_verbalizer: TN eletronic verbalizer diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/fraction.py index 8b3d60c04..df56d8d7f 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/fraction.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst, convert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst, convert_space + class FractionFst(GraphFst): """ @@ -24,7 +25,7 @@ class FractionFst(GraphFst): e.g. halv -> tokens { name: "1/2" } e.g. ett och en halv -> tokens { name: "1 1/2" } e.g. tre och fyra femtedelar -> tokens { name: "3 4/5" } - + Args: itn_cardinal_tagger: ITN cardinal tagger tn_fraction_verbalizer: TN fraction verbalizer diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/ordinal.py index 3f97c7ef5..bc91a7427 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/telephone.py index 7436407b3..74369e70f 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/telephone.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst, convert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst, convert_space + class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone numbers, e.g. + Finite state transducer for classifying telephone numbers, e.g. noll åtta sjuhundraåttionio femtiotvå tjugofem -> tokens { name: "08-789 52 25" } - + Args: tn_cardinal_tagger: TN Cardinal Tagger """ diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/time.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/time.py index 77d59aacb..311c14c36 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/time.py @@ -15,11 +15,12 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.sv.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst from nemo_text_processing.text_normalization.sv.utils import get_abs_path as get_tn_abs_path from nemo_text_processing.text_normalization.sv.utils import load_labels -from pynini.lib import pynutil QUARTERS = {15: "kvart över", 30: "halv", 45: "kvart i"} @@ -61,7 +62,7 @@ class TimeFst(GraphFst): e.g. klockan tretton tio -> time { hours: "kl. 13" minutes: "10" } e.g. kvart i tolv -> time { minutes: "45" hours: "11" } e.g. kvart över tolv -> time { minutes: "15" hours: "12" } - + Args: tn_cardinal_tagger: TN cardinal verbalizer """ diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/tokenize_and_classify.py index 9c8b26145..f8eef59d3 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.taggers.punctuation import PunctuationFst from nemo_text_processing.inverse_text_normalization.en.taggers.word import WordFst from nemo_text_processing.inverse_text_normalization.sv.taggers.cardinal import CardinalFst @@ -42,7 +43,7 @@ from nemo_text_processing.text_normalization.sv.taggers.ordinal import OrdinalFst as TNOrdinalTagger from nemo_text_processing.text_normalization.sv.taggers.telephone import TelephoneFst as TNTelephoneTagger from nemo_text_processing.text_normalization.sv.verbalizers.electronic import ElectronicFst as TNElectronicVerbalizer -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -73,9 +74,9 @@ def __init__( far_file = os.path.join(cache_dir, f"sv_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") tn_cardinal_tagger = TNCardinalTagger(deterministic=False) tn_ordinal_tagger = TNOrdinalTagger(cardinal=tn_cardinal_tagger, deterministic=False) tn_date_tagger = TNDateTagger(cardinal=tn_cardinal_tagger, ordinal=tn_ordinal_tagger, deterministic=False) @@ -136,4 +137,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/sv/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/sv/taggers/whitelist.py index df88a65a8..97a2fcf3b 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/sv/taggers/whitelist.py @@ -16,6 +16,8 @@ import os +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.sv.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_LOWER_CASED, @@ -23,7 +25,6 @@ convert_space, string_map_cased, ) -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/sv/utils.py b/nemo_text_processing/inverse_text_normalization/sv/utils.py index 0a7f1ff2d..e645db2dd 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/utils.py +++ b/nemo_text_processing/inverse_text_normalization/sv/utils.py @@ -21,7 +21,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/cardinal.py index 92a83625a..b13382a8e 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/date.py index 55ee362e3..e5c3f99e2 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SPACE, @@ -20,7 +22,6 @@ delete_preserve_order, delete_space, ) -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/decimal.py index 48b22dbfa..d3ab30c74 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -20,7 +22,6 @@ delete_space, ) from nemo_text_processing.text_normalization.sv.graph_utils import ensure_space -from pynini.lib import pynutil class DecimalFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/time.py index f8489ba7b..81aba6d98 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, @@ -20,7 +22,6 @@ GraphFst, delete_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/verbalize_final.py index 61b6433e3..643017c47 100644 --- a/nemo_text_processing/inverse_text_normalization/sv/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/sv/verbalizers/verbalize_final.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.inverse_text_normalization.sv.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "klockan" } tokens { name: "är" } tokens { time { hours: "12" minutes: "30" } } -> klockan är 12:30 """ diff --git a/nemo_text_processing/inverse_text_normalization/vi/data/currency.tsv b/nemo_text_processing/inverse_text_normalization/vi/data/currency.tsv index ce65d4420..0e14edf4f 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/data/currency.tsv +++ b/nemo_text_processing/inverse_text_normalization/vi/data/currency.tsv @@ -8,4 +8,4 @@ $ đô la mỹ ₩ won ₩ uôn RM ringgit -₫ đồng \ No newline at end of file +£ bảng anh \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/vi/data/electronic/symbols.tsv b/nemo_text_processing/inverse_text_normalization/vi/data/electronic/symbols.tsv index eccbe3d47..f8f2fc3d2 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/data/electronic/symbols.tsv +++ b/nemo_text_processing/inverse_text_normalization/vi/data/electronic/symbols.tsv @@ -2,6 +2,7 @@ - gạch _ gạch dưới _ shift gạch +_ shift trừ _ síp gạch ! chấm than # thăng diff --git a/nemo_text_processing/inverse_text_normalization/vi/data/electronic/url_symbols.tsv b/nemo_text_processing/inverse_text_normalization/vi/data/electronic/url_symbols.tsv new file mode 100644 index 000000000..99f2059a2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/vi/data/electronic/url_symbols.tsv @@ -0,0 +1,8 @@ +. chấm +- gạch +- gạch ngang +_ gạch dưới +_ shift gạch +_ shift trừ +_ síp gạch +/ sẹc diff --git a/nemo_text_processing/inverse_text_normalization/vi/data/numbers/digit_special.tsv b/nemo_text_processing/inverse_text_normalization/vi/data/numbers/digit_special.tsv new file mode 100644 index 000000000..ff6d70710 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/vi/data/numbers/digit_special.tsv @@ -0,0 +1,3 @@ +mốt 1 +tư 4 +lăm 5 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/vi/data/time/zero_prefix.tsv b/nemo_text_processing/inverse_text_normalization/vi/data/time/zero_prefix.tsv new file mode 100644 index 000000000..14d52dc0f --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/vi/data/time/zero_prefix.tsv @@ -0,0 +1,3 @@ +linh 0 +lẻ 0 +không 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/vi/graph_utils.py b/nemo_text_processing/inverse_text_normalization/vi/graph_utils.py index 5af266ad6..33c9700c3 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/vi/graph_utils.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import string from pathlib import Path @@ -24,6 +23,8 @@ from pynini.export import export from pynini.lib import byte, pynutil, utf8 +from nemo_text_processing.utils.logging import logger + NEMO_CHAR = utf8.VALID_UTF8_CHAR NEMO_DIGIT = byte.DIGIT @@ -32,11 +33,11 @@ NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() NEMO_HEX = pynini.union(*string.hexdigits).optimize() -NEMO_NON_BREAKING_SPACE = "\u00A0" +NEMO_NON_BREAKING_SPACE = "\u00a0" NEMO_SPACE = " " -NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize() +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() -NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, '"').optimize() NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() @@ -46,6 +47,7 @@ delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) insert_space = pynutil.insert(" ") delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_single_space = pynutil.delete(NEMO_SPACE) # French frequently compounds numbers with hyphen. delete_hyphen = pynutil.delete(pynini.closure("-", 0, 1)) @@ -67,7 +69,7 @@ def generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]): for rule, graph in graphs.items(): exporter[rule] = graph.optimize() exporter.close() - logging.info(f"Created {file_name}") + logger.info(f"Created {file_name}") def convert_space(fst) -> "pynini.FstLike": @@ -150,4 +152,4 @@ def delete_tokens(self, fst) -> "pynini.FstLike": + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/cardinal.py index d25c96071..bdc1b1c5f 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/cardinal.py @@ -15,6 +15,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( NEMO_DIGIT, NEMO_SPACE, @@ -22,7 +24,6 @@ delete_space, ) from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path -from pynini.lib import pynutil class CardinalFst(GraphFst): @@ -35,116 +36,122 @@ class CardinalFst(GraphFst): def __init__(self): super().__init__(name="cardinal", kind="classify") - graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) - graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + thousand_words = pynini.union("ngàn", "nghìn") + negative_words = pynini.union("âm", "trừ") + + graph_hundred = pynini.cross("trăm", "") + graph_ten = pynini.cross("mươi", "") + zero = pynini.cross(pynini.union("linh", "lẻ"), "0") + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) graph_one = pynini.cross("mốt", "1") graph_four = pynini.cross("tư", "4") graph_five = pynini.cross("lăm", "5") graph_half = pynini.cross("rưỡi", "5") - graph_hundred = pynini.cross("trăm", "") - graph_ten = pynini.cross("mươi", "") - zero = pynini.cross(pynini.union("linh", "lẻ"), "0") optional_ten = pynini.closure(delete_space + graph_ten, 0, 1) last_digit_exception = pynini.project(pynini.cross("năm", "5"), "input") - last_digit = pynini.union( + self.last_digit = pynini.union( (pynini.project(graph_digit, "input") - last_digit_exception.arcsort()) @ graph_digit, graph_one, graph_four, graph_five, ) - - graph_hundred_ties_component = (graph_digit | graph_zero) + delete_space + graph_hundred - graph_hundred_ties_component += delete_space - graph_hundred_ties_component += pynini.union( + last_digit = self.last_digit + # Build hundreds component (e.g., "một trăm", "hai trăm") + graph_hundreds_component = (graph_digit | graph_zero) + delete_space + graph_hundred + graph_hundreds_component += delete_space + graph_hundreds_component += pynini.union( graph_teen, - (graph_half | graph_four | graph_one) + pynutil.insert("0"), - graph_ties + optional_ten + ((delete_space + last_digit) | pynutil.insert("0")), - zero + delete_space + (graph_digit | graph_four), - pynutil.insert("00"), - ) - graph_hundred_ties_component |= ( + (graph_half | graph_four | graph_one) + pynutil.insert("0", weight=0.1), + graph_ties + optional_ten + ((delete_space + last_digit) | pynutil.insert("0", weight=0.1)), + zero + delete_space + (graph_digit | graph_four | graph_five), + pynutil.insert("00", weight=0.1), + ).optimize() + graph_hundreds_component |= ( pynutil.insert("0") + delete_space + pynini.union( graph_teen, graph_ties + optional_ten + delete_space + last_digit, - graph_ties + delete_space + graph_ten + pynutil.insert("0"), - zero + delete_space + (graph_digit | graph_four), - ) + graph_ties + delete_space + graph_ten + pynutil.insert("0", weight=0.1), + zero + delete_space + (graph_digit | graph_four | graph_five), + ).optimize() + ) + graph_hundred_component = graph_hundreds_component | ( + pynutil.insert("00", weight=0.1) + delete_space + graph_digit ) - graph_hundred_component = graph_hundred_ties_component | (pynutil.insert("00") + delete_space + graph_digit) graph_hundred_component_at_least_one_none_zero_digit = graph_hundred_component @ ( pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT) ) self.graph_hundred_component_at_least_one_none_zero_digit = ( - graph_hundred_component_at_least_one_none_zero_digit + graph_hundred_component_at_least_one_none_zero_digit.optimize() ) - graph_hundred_ties_zero = graph_hundred_ties_component | pynutil.insert("000") + graph_hundreds_zero = graph_hundreds_component | pynutil.insert("000", weight=0.1) graph_thousands = pynini.union( - graph_hundred_component_at_least_one_none_zero_digit - + delete_space - + pynutil.delete(pynini.union("nghìn", "ngàn")), + graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete(thousand_words), pynutil.insert("000", weight=0.1), - ) - - graph_ten_thousand = pynini.union( - graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("vạn"), - pynutil.insert("0000", weight=0.1), - ) - - graph_ten_thousand_suffix = pynini.union( - graph_digit + delete_space + pynutil.delete(pynini.union("nghìn", "ngàn")), - pynutil.insert("0", weight=0.1), - ) + ).optimize() graph_million = pynini.union( graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete("triệu"), pynutil.insert("000", weight=0.1), - ) + ).optimize() graph_billion = pynini.union( graph_hundred_component_at_least_one_none_zero_digit + delete_space + pynutil.delete(pynini.union("tỉ", "tỷ")), pynutil.insert("000", weight=0.1), - ) + ).optimize() + # Main graph combining all magnitude levels graph = pynini.union( + # Full format: billion + million + thousand + hundred graph_billion + delete_space + graph_million + delete_space + graph_thousands + delete_space - + graph_hundred_ties_zero, - graph_ten_thousand + delete_space + graph_ten_thousand_suffix + delete_space + graph_hundred_ties_zero, + + graph_hundreds_zero, + # Special thousand format with last digit or "rưỡi" (half) graph_hundred_component_at_least_one_none_zero_digit + delete_space - + pynutil.delete(pynini.union("nghìn", "ngàn")) + + pynutil.delete(thousand_words) + delete_space - + (((last_digit | graph_half) + pynutil.insert("00")) | graph_hundred_ties_zero), + + pynini.union( + pynini.union(last_digit, graph_half) + pynutil.insert("00", weight=0.1), graph_hundreds_zero + ), + # Single digits (for non-exception cases) graph_digit, graph_zero, ) - graph = graph @ pynini.union( - pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), "0", + graph = ( + graph + @ pynini.union( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), + "0", + ).optimize() ) # don't convert cardinals from zero to nine inclusive - graph_exception = pynini.project(pynini.union(graph_digit, graph_zero), "input") + single_digits = pynini.project(pynini.union(graph_digit, graph_zero), "input").optimize() self.graph_no_exception = graph - self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + self.graph = pynini.difference(pynini.project(graph, "input"), single_digits) @ graph optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross(pynini.union("âm", "trừ"), '"-"') + NEMO_SPACE, 0, 1, + pynutil.insert("negative: ") + pynini.cross(negative_words, '"-"') + NEMO_SPACE, + 0, + 1, ) final_graph = optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/date.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/date.py index e1986f69c..8292bcfe0 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/date.py @@ -14,144 +14,161 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space -from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path from pynini.lib import pynutil -graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).optimize() -graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize() -graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize() -ties_graph = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).optimize() +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path -def _get_month_graph(): - """ - Transducer for month, e.g. march -> march +class DateFst(GraphFst): """ - month_graph = pynini.string_file(get_abs_path("data/months.tsv")).optimize() - return month_graph - + Finite state transducer for classifying date, + e.g. mười lăm tháng một năm hai nghìn mười hai -> date { day: "15" month: "1" year: "2012" preserve_order: true } + e.g. ngày ba mốt tháng mười hai năm một chín chín chín -> date { day: "31" month: "12" year: "2012" preserve_order: true } + e.g. năm hai không hai mốt -> date { year: "2021" preserve_order: true } -def _get_ties_graph(): - """ - Transducer for 20-99 e.g - hai ba -> 23 + Args: + cardinal: CardinalFst """ - graph_one = pynini.cross("mốt", "1") - graph_four = pynini.cross("tư", "4") - graph_five = pynini.cross("lăm", "5") - graph_ten = pynini.cross("mươi", "") - optional_ten = pynini.closure(delete_space + graph_ten, 0, 1) - graph = pynini.union( - ties_graph + optional_ten + delete_space + (graph_digit | graph_one | graph_four | graph_five), - ties_graph + delete_space + graph_ten + pynutil.insert("0"), - ) - return graph + def __init__(self, cardinal: GraphFst): + super().__init__(name="date", kind="classify") + cardinal_graph = cardinal.graph_no_exception -def _get_year_graph(): - """ - Transducer for year, e.g. hai không hai mươi -> 2020 - """ + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).optimize() + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize() + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize() + ties_graph = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).optimize() + + # Special digit mappings for Vietnamese + graph_one = pynini.cross("mốt", "1") + graph_four = pynini.cross("tư", "4") + graph_five = pynini.cross("lăm", "5") + graph_ten = pynini.cross("mươi", "") + optional_ten = pynini.closure(delete_space + graph_ten, 0, 1) + + # Ties graph for 20-99 (e.g., "hai ba" -> "23") + graph_ties = pynini.union( + ties_graph + optional_ten + delete_space + pynini.union(graph_digit, graph_one, graph_four, graph_five), + ties_graph + delete_space + graph_ten + pynutil.insert("0", weight=0.01), + ) - def _get_digits_graph(): + # Zero prefix patterns (e.g., "linh năm" -> "05") zero = pynini.cross((pynini.union("linh", "lẻ")), "0") - four = pynini.cross("tư", "4") - graph = pynini.union(zero + delete_space + (graph_digit | four), graph_zero + delete_space + graph_digit,) - graph.optimize() - return graph - - def _get_hundreds_graph(graph_ties, graph_digits): - graph = ( + graph_digits = pynini.union( + zero + delete_space + pynini.union(graph_digit, graph_four), + graph_zero + delete_space + graph_digit, + ).optimize() + + # Year components + # Hundreds pattern (e.g., "hai trăm mười hai" -> "212") + year_hundreds = ( graph_digit + delete_space + pynutil.delete("trăm") + delete_space - + (graph_teen | graph_ties | graph_digits) + + pynini.union(graph_teen, graph_ties, graph_digits) ) - return graph - def _get_thousands_graph(graph_ties, graph_digits): - graph_hundred_component = ( - (graph_digit | graph_zero) + delete_space + pynutil.delete("trăm") - ) | pynutil.insert("0") - graph = ( + # Thousands pattern with optional hundreds (e.g., "hai nghìn không ba" -> "2003") + year_hundred_component = pynini.union( + pynini.union(graph_digit, graph_zero) + delete_space + pynutil.delete("trăm"), + pynutil.insert("0", weight=0.01), + ) + year_thousands = ( graph_digit + delete_space + pynutil.delete(pynini.union("nghìn", "ngàn")) + delete_space - + graph_hundred_component + + year_hundred_component + delete_space - + (graph_teen | graph_ties | graph_digits) + + pynini.union(graph_teen, graph_ties, graph_digits) ) - return graph - - graph_ties = _get_ties_graph() - graph_digits = _get_digits_graph() - graph_hundreds = _get_hundreds_graph(graph_ties, graph_digits) - graph_thousands = _get_thousands_graph(graph_ties, graph_digits) - year_graph = ( - # 20 19, 40 12, 2012, 2 0 0 5, 2 0 17, 938 - assuming no limit on the year - graph_digit - + delete_space - + (graph_digit | graph_zero) - + delete_space - + (graph_teen | graph_ties | graph_digits) - | graph_thousands - | graph_hundreds - | (graph_digit + pynutil.insert("0") + delete_space + (graph_ties | graph_digits | graph_teen)) - ) - year_graph.optimize() - return year_graph - - -class DateFst(GraphFst): - """ - Finite state transducer for classifying date, - e.g. mười lăm tháng một năm hai nghìn mười hai -> date { day: "15" month: "1" year: "2012" preserve_order: true } - e.g. ngày ba mốt tháng mười hai năm một chín chín chín -> date { day: "31" month: "12" year: "2012" preserve_order: true } - e.g. năm hai không hai mốt -> date { year: "2021" preserve_order: true } - Args: - cardinal: CardinalFst - """ + # Standard XYYZ pattern (e.g., "hai không một chín" -> "2019") + year_standard = ( + graph_digit + + delete_space + + pynini.union(graph_digit, graph_zero) + + delete_space + + pynini.union(graph_teen, graph_ties, graph_digits) + ) - def __init__(self, cardinal: GraphFst): - super().__init__(name="date", kind="classify") + # XYZ pattern with implied 0 (e.g., "hai không hai mốt" -> "2021") + year_implied_zero = ( + graph_digit + + pynutil.insert("0", weight=0.01) + + delete_space + + pynini.union(graph_ties, graph_digits, graph_teen) + ) - cardinal_graph = cardinal.graph_no_exception - year_graph = _get_year_graph() - YEAR_WEIGHT = 0.001 - year_graph = pynutil.add_weight(year_graph, YEAR_WEIGHT) - month_graph = _get_month_graph() + # Digit-by-digit pattern (e.g., "hai không một chín" -> "2019") + year_digit_by_digit = ( + pynini.union(graph_digit, graph_zero) + + delete_space + + pynini.union(graph_digit, graph_zero) + + delete_space + + pynini.union(graph_digit, graph_zero) + + delete_space + + pynini.union(graph_digit, graph_zero) + ) - month_graph = pynutil.insert('month: "') + month_graph + pynutil.insert('"') + year_graph = pynini.union( + year_standard, + year_thousands, + year_hundreds, + year_implied_zero, + year_digit_by_digit, + ).optimize() + + # Month graph with special handling for "năm" (means "5" in months but "year" in other contexts) + month_graph = ( + pynutil.insert('month: "') + + pynini.string_file(get_abs_path("data/months.tsv")).optimize() + + pynutil.insert('"') + ) month_exception = pynini.project(pynini.cross("năm", "5"), "input") month_graph_exception = (pynini.project(month_graph, "input") - month_exception.arcsort()) @ month_graph day_graph = pynutil.insert('day: "') + cardinal_graph + pynutil.insert('"') - # day_suffix = pynini.union("ngày", "mùng") - # optional_day = pynini.closure(day_suffix + delete_space, 0, 1) - graph_month = pynutil.delete("tháng") + delete_space + month_graph_exception - graph_year = ( + + graph_year = pynutil.add_weight( delete_extra_space + pynutil.delete("năm") + delete_extra_space + pynutil.insert('year: "') - + pynutil.add_weight(year_graph, -YEAR_WEIGHT) - + pynutil.insert('"') + + year_graph + + pynutil.insert('"'), + -0.1, ) - optional_graph_year = pynini.closure(graph_year, 0, 1) - graph_my = pynutil.delete("tháng") + delete_space + month_graph + graph_year + + # Date pattern combinations + # Pattern 1: Day-Month-Year (e.g., "ngày 15 tháng 1 năm 2024") graph_dmy = ( - day_graph + delete_space + pynutil.delete("tháng") + delete_extra_space + month_graph + optional_graph_year + day_graph + + delete_space + + pynutil.delete("tháng") + + delete_extra_space + + month_graph + + pynini.closure(graph_year, 0, 1) ) - graph_year = ( - pynutil.delete("năm") + delete_extra_space + pynutil.insert('year: "') + year_graph + pynutil.insert('"') + + # Pattern 2: Month-Year (e.g., "tháng 1 năm 2024") + graph_my = pynutil.delete("tháng") + delete_space + month_graph + graph_year + + # Pattern 3: Standalone year (e.g., "năm 2024") + graph_year_standalone = pynutil.add_weight( + pynutil.delete("năm") + delete_extra_space + pynutil.insert('year: "') + year_graph + pynutil.insert('"'), + -0.1, ) - final_graph = (graph_dmy | graph_my | graph_month | graph_year) + pynutil.insert(" preserve_order: true") - final_graph = self.add_tokens(final_graph) - self.fst = final_graph.optimize() + final_graph = pynini.union( + graph_dmy, # Day-Month-Year + graph_my, # Month-Year + graph_month, # Month only + graph_year_standalone, # Year only + ) + pynutil.insert(" preserve_order: true") + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/decimal.py index 298319306..7c2bd2151 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/decimal.py @@ -14,69 +14,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( NEMO_DIGIT, GraphFst, delete_extra_space, delete_space, + insert_space, ) from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path -from pynini.lib import pynutil - -graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) - - -def get_quantity(decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike") -> "pynini.FstLike": - """ - Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, - e.g. một triệu -> integer_part: "1" quantity: "triệu" - e.g. một tỷ rưỡi -> integer_part: "1" fractional_part: "5" quantity: "tỷ" - - Args: - decimal: decimal FST - cardinal_up_to_hundred: cardinal FST - """ - numbers = cardinal_up_to_hundred @ ( - pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) - ) - suffix = pynini.union("triệu", "tỉ", "tỷ", "vạn") - graph_four = pynini.cross("tư", "4") - graph_one = pynini.cross("mốt", "1") - graph_half = pynini.cross("rưỡi", "5") - last_digit_exception = pynini.project(pynini.cross("năm", "5"), "input") - last_digit = pynini.union( - (pynini.project(graph_digit, "input") - last_digit_exception.arcsort()) @ graph_digit, - graph_one, - graph_four, - graph_half, - ) - optional_fraction_graph = pynini.closure( - delete_extra_space - + pynutil.insert('fractional_part: "') - + (last_digit | graph_half | graph_one | graph_four) - + pynutil.insert('"'), - 0, - 1, - ) - - res = ( - pynutil.insert('integer_part: "') - + numbers - + pynutil.insert('"') - + delete_extra_space - + pynutil.insert('quantity: "') - + suffix - + pynutil.insert('"') - + optional_fraction_graph - ) - res |= ( - decimal - + delete_extra_space - + pynutil.insert('quantity: "') - + (suffix | "ngàn" | "nghìn") - + pynutil.insert('"') - ) - return res class DecimalFst(GraphFst): @@ -94,38 +41,88 @@ def __init__(self, cardinal: GraphFst): cardinal_graph = cardinal.graph_no_exception - graph_decimal = graph_digit | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) - graph_one = pynini.cross("mốt", "1") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + base_decimal = graph_digit | graph_zero graph_four = pynini.cross("tư", "4") graph_five = pynini.cross("lăm", "5") + graph_one = pynini.cross("mốt", "1") + negative_words = pynini.union("âm", "trừ") graph_decimal = pynini.union( - graph_decimal, + base_decimal, graph_four, - pynini.closure(graph_decimal + delete_space, 1) + (graph_decimal | graph_four | graph_five | graph_one), - ) + pynini.closure(base_decimal + delete_space, 1) + (base_decimal | graph_four | graph_five | graph_one), + ).optimize() self.graph = graph_decimal point = pynutil.delete("chấm") | pynutil.delete("phẩy") - optional_graph_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross(pynini.union("âm", "trừ"), '"true"') + delete_extra_space, + pynutil.insert("negative:") + insert_space + pynini.cross(negative_words, '"true"') + delete_extra_space, 0, 1, ) - graph_fractional = pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"') + graph_fractional = ( + pynutil.insert('fractional_part:') + + insert_space + + pynutil.insert('"') + + graph_decimal + + pynutil.insert('"') + ) graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') final_graph_wo_sign = ( pynini.closure(graph_integer + delete_extra_space, 0, 1) + point + delete_extra_space + graph_fractional ) - final_graph = optional_graph_negative + final_graph_wo_sign + # Build quantity handling - reuse magnitude words from cardinal context + # e.g. một triệu -> integer_part: "1" quantity: "triệu" + # e.g. một tỷ rưỡi -> integer_part: "1" fractional_part: "5" quantity: "tỷ" + numbers = cardinal.graph_hundred_component_at_least_one_none_zero_digit @ ( + pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT) + ) + + magnitude_words = pynini.union("triệu", "tỉ", "tỷ", "vạn") + thousand_words = pynini.union("ngàn", "nghìn") + + last_digit = cardinal.last_digit + optional_fraction_graph = pynini.closure( + delete_extra_space + + pynutil.insert('fractional_part:') + + insert_space + + pynutil.insert('"') + + (last_digit | pynini.cross("rưỡi", "5") | graph_one | graph_four) + + pynutil.insert('"'), + 0, + 1, + ) - self.final_graph_wo_negative = final_graph_wo_sign | get_quantity( - final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit, + quantity_graph = ( + pynutil.insert('integer_part:') + + insert_space + + pynutil.insert('"') + + numbers + + pynutil.insert('"') + + delete_extra_space + + pynutil.insert('quantity:') + + insert_space + + pynutil.insert('"') + + magnitude_words + + pynutil.insert('"') + + optional_fraction_graph ) - final_graph |= optional_graph_negative + get_quantity( - final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit, + quantity_graph |= ( + final_graph_wo_sign + + delete_extra_space + + pynutil.insert('quantity:') + + insert_space + + pynutil.insert('"') + + (magnitude_words | thousand_words) + + pynutil.insert('"') ) + + final_graph = optional_graph_negative + final_graph_wo_sign + + self.final_graph_wo_negative = final_graph_wo_sign | quantity_graph + final_graph |= optional_graph_negative + quantity_graph final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/electronic.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/electronic.py index 3eb0c8886..9c641597e 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/electronic.py @@ -14,10 +14,16 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_ALPHA, GraphFst, insert_space -from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( + NEMO_ALPHA, + GraphFst, + delete_single_space, + insert_space, +) +from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path + class ElectronicFst(GraphFst): """ @@ -28,66 +34,77 @@ class ElectronicFst(GraphFst): def __init__(self): super().__init__(name="electronic", kind="classify") - delete_extra_space = pynutil.delete(" ") - alpha_num = ( - NEMO_ALPHA - | pynini.string_file(get_abs_path("data/numbers/digit.tsv")) - | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + alpha_num = pynini.union( + NEMO_ALPHA, + pynini.string_file(get_abs_path("data/numbers/digit.tsv")), + pynini.string_file(get_abs_path("data/numbers/zero.tsv")), ) symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert() + url_symbols = pynini.string_file(get_abs_path("data/electronic/url_symbols.tsv")).invert() - accepted_username = alpha_num | symbols + accepted_username = pynini.union(alpha_num, symbols) + accepted_url_chars = pynini.union(alpha_num, url_symbols) process_dot = pynini.cross("chấm", ".") username = ( pynutil.insert('username: "') + alpha_num - + pynini.closure(delete_extra_space + accepted_username) + + pynini.closure(delete_single_space + accepted_username) + pynutil.insert('"') ) - single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num - server = single_alphanum | pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) - domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + single_alphanum = pynini.closure(alpha_num + delete_single_space) + alpha_num + server = pynini.union( + single_alphanum, + pynini.string_file(get_abs_path("data/electronic/server_name.tsv")), + pynini.closure(NEMO_ALPHA, 2), # At least 2 letters for server name + ) + domain = pynini.union( + single_alphanum, + pynini.string_file(get_abs_path("data/electronic/domain.tsv")), + pynini.closure(NEMO_ALPHA, 2), # At least 2 letters for domain + ) multi_domain = ( - pynini.closure(process_dot + delete_extra_space + domain + delete_extra_space) + pynini.closure(process_dot + delete_single_space + domain + delete_single_space) + process_dot - + delete_extra_space + + delete_single_space + domain ) - domain_graph = pynutil.insert('domain: "') + server + delete_extra_space + multi_domain + pynutil.insert('"') + domain_graph = pynutil.insert('domain: "') + server + delete_single_space + multi_domain + pynutil.insert('"') graph = ( username - + delete_extra_space + + delete_single_space + pynutil.delete(pynini.union("a còng", "a móc", "a vòng")) + insert_space - + delete_extra_space + + delete_single_space + domain_graph ) - ############# url ### protocol_end = pynini.cross(pynini.union("w w w", "www"), "www") - protocol_start = (pynini.cross("h t t p", "http") | pynini.cross("h t t p s", "https")) + pynini.cross( - " hai chấm sẹc sẹc ", "://" - ) - # .com, - ending = ( - delete_extra_space - + symbols - + delete_extra_space - + (domain | pynini.closure(accepted_username + delete_extra_space) + accepted_username) - ) + protocol_start = pynini.union( + pynini.cross("h t t p", "http"), pynini.cross("h t t p s", "https") + ) + pynini.cross(" hai chấm sẹc sẹc ", "://") + + # Domain part: server.domain (e.g., nvidia.com, www.nvidia.com) + url_domain = server + delete_single_space + process_dot + delete_single_space + domain + # Optional endings: /path or .vn or .com.vn + url_ending = ( + delete_single_space + + url_symbols + + delete_single_space + + pynini.union(domain, pynini.closure(accepted_url_chars + delete_single_space) + accepted_url_chars) + ) protocol = ( - pynini.closure(protocol_start, 0, 1) - + protocol_end - + delete_extra_space - + process_dot - + pynini.closure(delete_extra_space + accepted_username, 1) - + pynini.closure(ending, 1, 2) + pynini.closure(protocol_start, 0, 1) # Optional http:// + + pynini.closure( + protocol_end + delete_single_space + process_dot + delete_single_space, 0, 1 + ) # Optional www. + + url_domain # Required: server.domain + + pynini.closure(url_ending, 0) # Optional: /path or .vn ) + protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"') - graph |= protocol - ######## + graph = pynini.union(graph, protocol) final_graph = self.add_tokens(graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/fraction.py index 7798be654..4fe845569 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/fraction.py @@ -14,9 +14,15 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) + class FractionFst(GraphFst): """ @@ -31,14 +37,21 @@ class FractionFst(GraphFst): def __init__(self, cardinal: GraphFst): super().__init__(name="fraction", kind="classify") - # integer_part # numerator # denominator graph_cardinal = cardinal.graph_no_exception graph_four = pynini.cross("tư", "4") - numerator = pynutil.insert('numerator: "') + graph_cardinal + pynutil.insert('"') + numerator = ( + pynutil.insert('numerator:') + insert_space + pynutil.insert('"') + graph_cardinal + pynutil.insert('"') + ) fraction_component = pynutil.delete(pynini.union("phần", "trên", "chia")) - denominator = pynutil.insert('denominator: "') + (graph_cardinal | graph_four) + pynutil.insert('"') + denominator = ( + pynutil.insert('denominator:') + + insert_space + + pynutil.insert('"') + + (graph_cardinal | graph_four) + + pynutil.insert('"') + ) graph_fraction_component = numerator + delete_space + fraction_component + delete_extra_space + denominator self.graph_fraction_component = graph_fraction_component @@ -48,7 +61,10 @@ def __init__(self, cardinal: GraphFst): self.final_graph_wo_negative = graph optional_graph_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross(pynini.union("âm", "trừ"), '"true"') + delete_extra_space, + pynutil.insert("negative:") + + insert_space + + pynini.cross(pynini.union("âm", "trừ"), '"true"') + + delete_extra_space, 0, 1, ) diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/measure.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/measure.py index 0b1f08b87..fef7d6d93 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/measure.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( GraphFst, convert_space, @@ -21,7 +23,6 @@ delete_space, ) from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path -from pynini.lib import pynutil class MeasureFst(GraphFst): @@ -58,7 +59,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): unit_singular = ( pynutil.insert('units: "') - + (unit_singular | unit_misc | pynutil.add_weight(unit_singular + delete_space + unit_misc, 0.01)) + + pynini.union( + unit_singular, unit_misc, pynutil.add_weight(unit_singular + delete_space + unit_misc, 0.01) + ) + pynutil.insert('"') ) @@ -84,20 +87,23 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): fraction_graph = ( delete_extra_space + pynutil.insert('fractional_part: "') - + (graph_digit | graph_half | graph_one | graph_four) + + pynini.union(graph_digit, graph_half, graph_one, graph_four) + pynutil.insert('"') ) - subgraph_cardinal |= ( - pynutil.insert("cardinal { ") - + optional_graph_negative - + pynutil.insert('integer: "') - + cardinal_graph - + pynutil.insert('" }') - + delete_extra_space - + unit_singular - + fraction_graph + subgraph_cardinal = pynini.union( + subgraph_cardinal, + ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert('integer: "') + + cardinal_graph + + pynutil.insert('" }') + + delete_extra_space + + unit_singular + + fraction_graph + ), ) - final_graph = subgraph_decimal | subgraph_cardinal + final_graph = pynini.union(subgraph_decimal, subgraph_cardinal) final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/money.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/money.py index b278773da..b2aacc6e5 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/money.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( NEMO_DIGIT, GraphFst, @@ -21,7 +23,6 @@ delete_extra_space, ) from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path -from pynini.lib import pynutil class MoneyFst(GraphFst): @@ -48,13 +49,13 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): graph_unit_singular = pynutil.insert('currency: "') + convert_space(unit_singular) + pynutil.insert('"') - add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + add_leading_zero_to_double_digit = pynini.union((NEMO_DIGIT + NEMO_DIGIT), (pynutil.insert("0") + NEMO_DIGIT)) # twelve dollars fifty, only after integer optional_cents_suffix = pynini.closure( delete_extra_space + pynutil.insert('fractional_part: "') - + (pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7) | graph_half) + + pynini.union(pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7), graph_half) + pynutil.insert('"'), 0, 1, @@ -70,6 +71,6 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): ) graph_decimal = graph_decimal_final + delete_extra_space + graph_unit_singular + optional_cents_suffix - final_graph = graph_integer | graph_decimal + final_graph = pynini.union(graph_integer, graph_decimal) final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/ordinal.py index a7f79a1a8..62b1c1205 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/ordinal.py @@ -14,10 +14,11 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space -from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space, insert_space +from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path + class OrdinalFst(GraphFst): """ @@ -33,6 +34,14 @@ def __init__(self): graph = graph_digit self.graph = graph - final_graph = pynutil.insert('integer: "') + graph_ordinal + delete_space + self.graph + pynutil.insert('"') + final_graph = ( + pynutil.insert('integer:') + + insert_space + + pynutil.insert('"') + + graph_ordinal + + delete_space + + self.graph + + pynutil.insert('"') + ) final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/punctuation.py index cbfe13063..4c96e0d0a 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/punctuation.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/punctuation.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst + class PunctuationFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/telephone.py index 202547a40..19e5f6db4 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/telephone.py @@ -14,27 +14,115 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space -from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( + NEMO_DIGIT, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path + class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone numbers, e.g. - một hai ba một hai ba năm sáu bảy tám -> { number_part: "1231235678" } + Finite state transducer for classifying telephone numbers and IP addresses. + + Supported formats: + + 1. Basic telephone (Vietnamese mobile with formatting): + "không chín ba sáu năm năm năm bốn bốn chín" + -> telephone { number_part: "093-655-5449" } + + 2. International format with country code: + "cộng tám mươi bốn không chín ba sáu năm năm năm bốn bốn chín" + -> telephone { country_code: "+84" number_part: "093-655-5449" } + + 3. IP addresses (using "chấm" for dot): + "một chín hai chấm một sáu tám chấm không chấm một" + -> telephone { number_part: "192.168.0.1" } + + 4. Emergency/hotline numbers: + "một một hai" -> telephone { number_part: "112" } + + 5. Credit card (15-16 digits): + "một hai ba bốn năm sáu bảy tám chín mười một hai ba bốn năm sáu" + -> telephone { number_part: "1234 5678 9101 2345" } + + Args: + cardinal: CardinalFst - required for parsing multi-digit numbers like "tám mươi bốn" """ - def __init__(self): + def __init__(self, cardinal: GraphFst): super().__init__(name="telephone", kind="classify") + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) - digit = graph_digit | graph_zero - last_digit = digit | pynini.cross("mốt", "1") | pynini.cross("tư", "4") | pynini.cross("lăm", "5") + graph_digit_special = pynini.string_file(get_abs_path("data/numbers/digit_special.tsv")) + digit = pynini.union(graph_digit, graph_zero) + last_digit = pynini.union(digit, graph_digit_special) + cardinal_two_digit = pynini.compose(cardinal.graph_no_exception, NEMO_DIGIT**2) + + vietnamese_mobile = pynini.compose( + pynini.cross("không", "0") + delete_space + pynini.closure(digit + delete_space, 8) + last_digit, + pynini.accep("0") + + NEMO_DIGIT**2 + + pynutil.insert("-") + + NEMO_DIGIT**3 + + pynutil.insert("-") + + NEMO_DIGIT**4, + ) + + basic_digits = pynini.closure(digit + delete_space, 2) + last_digit + + basic_phone = pynini.union(pynutil.add_weight(vietnamese_mobile, -0.01), basic_digits) + + country_code_digits = pynini.union( + pynutil.add_weight(cardinal_two_digit, -0.001), digit + delete_space + digit, digit + ) + + phone_with_country_code = ( + pynutil.insert('country_code: "') + + pynini.cross("cộng ", "+") + + country_code_digits + + pynutil.insert('"') + + delete_space + + insert_space + + pynutil.insert('number_part: "') + + basic_phone + + pynutil.insert('"') + ) + + phone_basic = pynutil.insert('number_part: "') + basic_phone + pynutil.insert('"') + + basic_phone_graph = pynini.union(pynutil.add_weight(phone_with_country_code, -0.1), phone_basic) + + ip_octet = pynini.union(pynini.closure(digit + delete_space, 0, 2) + digit, cardinal_two_digit) + + ip_graph = ip_octet + (delete_space + pynini.cross("chấm", ".") + delete_space + ip_octet) ** 3 + + ip_with_tag = pynutil.insert('number_part: "') + ip_graph + pynutil.insert('"') + + sixteen_digits = pynini.closure(digit + delete_space, 15) + digit + card_16 = pynini.compose( + sixteen_digits, + NEMO_DIGIT**4 + insert_space + NEMO_DIGIT**4 + insert_space + NEMO_DIGIT**4 + insert_space + NEMO_DIGIT**4, + ) + + fifteen_digits = pynini.closure(digit + delete_space, 14) + digit + card_15 = pynini.compose( + fifteen_digits, NEMO_DIGIT**4 + insert_space + NEMO_DIGIT**6 + insert_space + NEMO_DIGIT**5 + ) + + card_graph = pynini.union(card_16, card_15) + card_with_tag = pynutil.insert('number_part: "') + card_graph + pynutil.insert('"') - graph_number_part = pynini.closure(digit + delete_space, 2) + last_digit - number_part = pynutil.insert('number_part: "') + graph_number_part + pynutil.insert('"') + graph = pynini.union( + pynutil.add_weight(ip_with_tag, weight=-0.01), + pynutil.add_weight(card_with_tag, weight=-0.005), + basic_phone_graph, + ) - graph = number_part final_graph = self.add_tokens(graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/time.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/time.py index c13d6467e..326306c76 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/time.py @@ -15,6 +15,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( GraphFst, convert_space, @@ -23,7 +25,6 @@ insert_space, ) from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -37,7 +38,6 @@ class TimeFst(GraphFst): def __init__(self): super().__init__(name="time", kind="classify") - # hours, minutes, seconds, suffix, zone, style, speak_period graph_hours_to = pynini.string_file(get_abs_path("data/time/hours_to.tsv")) graph_minutes_to = pynini.string_file(get_abs_path("data/time/minutes_to.tsv")) @@ -51,9 +51,17 @@ def __init__(self): optional_minute = pynini.closure(delete_space + minute, 0, 1) second = pynini.cross("giây", "") + # Zero prefix patterns for minutes (linh, lẻ, không) + # Examples: "linh năm" -> "05", "không tám" -> "08" + zero_prefix = pynini.string_file(get_abs_path("data/time/zero_prefix.tsv")) + graph_zero_minute = zero_prefix + delete_space + graph_minutes + graph_minute_extended = graph_minutes | graph_zero_minute + final_graph_hour = pynutil.insert('hours: "') + graph_hours + pynutil.insert('"') + delete_space + oclock - graph_minute = graph_minutes + optional_minute - graph_second = graph_minutes + delete_space + second + graph_minute = graph_minute_extended + optional_minute + graph_second = graph_minute_extended + delete_space + second + + # Optional time zone support final_time_zone_optional = pynini.closure( delete_space + insert_space @@ -64,6 +72,8 @@ def __init__(self): 1, ) + # Time pattern combinations + # Pattern 1: Hour + Minutes (e.g., "tám giờ hai mươi" -> 8:20) graph_hm = ( final_graph_hour + delete_extra_space @@ -72,6 +82,7 @@ def __init__(self): + pynutil.insert('"') ) + # Pattern 2: Hour + Minutes + Seconds (e.g., "tám giờ hai mươi phút ba mươi giây" -> 8:20:30) graph_hms = ( final_graph_hour + delete_extra_space @@ -86,6 +97,7 @@ def __init__(self): + pynutil.insert('"') ) + # Pattern 3: Minutes + Seconds only (e.g., "ba phút hai mươi giây" -> 3p20s) graph_ms = ( pynutil.insert('minutes: "') + graph_minutes @@ -98,9 +110,22 @@ def __init__(self): + pynutil.insert('"') ) + # Pattern 4: Hour + Seconds only (e.g., "ba giờ mười giây" -> 3:00:10) + graph_hs = ( + final_graph_hour + + delete_extra_space + + pynutil.insert('minutes: "0"') + + delete_extra_space + + pynutil.insert('seconds: "') + + graph_second + + pynutil.insert('"') + ) + + # "Kém" pattern components (e.g., "chín giờ kém hai mươi" -> 8:40) graph_hours_to_component = graph_hours @ graph_hours_to graph_minutes_to_component = graph_minutes @ graph_minutes_to + # Pattern 5: "Kém" time format (hour minus minutes) graph_time_to = ( pynutil.insert('hours: "') + graph_hours_to_component @@ -116,10 +141,18 @@ def __init__(self): + optional_minute ) - final_graph = (final_graph_hour | graph_hm | graph_hms) + final_time_zone_optional - final_graph |= graph_ms - final_graph |= graph_time_to + # Combine all time patterns + final_graph = ( + pynini.union( + final_graph_hour, # Hour only + graph_hm, # Hour + Minutes + graph_hms, # Hour + Minutes + Seconds + graph_hs, # Hour + Seconds + graph_ms, # Minutes + Seconds only + graph_time_to, # "Kém" pattern + ) + + final_time_zone_optional + ) final_graph = self.add_tokens(final_graph) - self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py index 068562ade..34ef1c5c0 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/tokenize_and_classify.py @@ -13,15 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( GraphFst, delete_extra_space, delete_space, generator_main, + insert_space, ) from nemo_text_processing.inverse_text_normalization.vi.taggers.cardinal import CardinalFst from nemo_text_processing.inverse_text_normalization.vi.taggers.date import DateFst @@ -37,7 +39,7 @@ from nemo_text_processing.inverse_text_normalization.vi.taggers.whitelist import WhiteListFst from nemo_text_processing.inverse_text_normalization.vi.taggers.word import WordFst from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -68,9 +70,9 @@ def __init__( far_file = os.path.join(cache_dir, f"vi_itn_{input_case}.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") cardinal = CardinalFst() cardinal_graph = cardinal.fst @@ -91,28 +93,26 @@ def __init__( whitelist_graph = WhiteListFst(input_file=whitelist).fst punct_graph = PunctuationFst().fst electronic_graph = ElectronicFst().fst - telephone_graph = TelephoneFst().fst + telephone_graph = TelephoneFst(cardinal=cardinal).fst classify = ( pynutil.add_weight(whitelist_graph, 1.01) - | pynutil.add_weight(time_graph, 1.05) - | pynutil.add_weight(date_graph, 1.09) - | pynutil.add_weight(decimal_graph, 1.08) + | pynutil.add_weight(time_graph, 1.09) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) | pynutil.add_weight(measure_graph, 1.1) | pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(ordinal_graph, 1.1) - | pynutil.add_weight(fraction_graph, 1.09) - | pynutil.add_weight(money_graph, 1.07) + | pynutil.add_weight(fraction_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) | pynutil.add_weight(telephone_graph, 1.1) - | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.11) | pynutil.add_weight(word_graph, 100) ) punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") - token_plus_punct = ( - pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) - ) + token_plus_punct = pynini.closure(punct + insert_space) + token + pynini.closure(insert_space + punct) graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) graph = delete_space + graph + delete_space @@ -121,4 +121,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/whitelist.py index 027a7af73..f9479d500 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/whitelist.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, convert_space from nemo_text_processing.inverse_text_normalization.vi.utils import get_abs_path -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/vi/taggers/word.py b/nemo_text_processing/inverse_text_normalization/vi/taggers/word.py index 3e1367d70..4ec96b077 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/vi/taggers/word.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py index 584758a3a..c1eda376a 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/date.py index d746528db..13e5db8d4 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DateFst(GraphFst): """ @@ -67,7 +68,9 @@ def __init__(self): + delete_space ) - final_graph = (graph_y | graph_m | graph_dm | graph_dmy | graph_my) + delete_space + optional_preserve_order + final_graph = ( + pynini.union(graph_y, graph_m, graph_dm, graph_dmy, graph_my) + delete_space + optional_preserve_order + ) delete_tokens = self.delete_tokens(final_graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py index c016c20e0..d25621f66 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/decimal.py @@ -13,9 +13,15 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_space, +) + class DecimalFst(GraphFst): """ @@ -50,7 +56,7 @@ def __init__(self): + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ) - optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1) + optional_quantity = pynini.closure(pynutil.insert(NEMO_SPACE) + quantity + delete_space, 0, 1) graph = optional_integer + optional_fractional + optional_quantity self.numbers = graph graph = optionl_sign + graph diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py index 755172492..443051a73 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class ElectronicFst(GraphFst): """ @@ -49,7 +50,7 @@ def __init__(self): ) graph = user_name + delete_space + pynutil.insert("@") + domain - graph |= protocol + graph = pynini.union(graph, protocol) delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py index e5e87767b..b19502446 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/fraction.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/measure.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/measure.py index abc06f724..6342bc1e9 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/measure.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/measure.py @@ -14,13 +14,15 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( - NEMO_CHAR, NEMO_NOT_QUOTE, + NEMO_NOT_SPACE, GraphFst, delete_space, + insert_space, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): @@ -36,11 +38,23 @@ class MeasureFst(GraphFst): def __init__(self, decimal: GraphFst, cardinal: GraphFst): super().__init__(name="measure", kind="verbalize") optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1) - unit = ( + # Units that don't need space (time units) + no_space_units = pynini.union("s", "ms", "ns", "μs", "h", "min", "%") + + unit_no_space = ( pynutil.delete("units:") + delete_space + pynutil.delete('"') - + pynini.closure(NEMO_CHAR - " ", 1) + + no_space_units + + pynutil.delete('"') + + delete_space + ) + + unit_with_space = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete('"') + + (pynini.closure(NEMO_NOT_SPACE, 1) - no_space_units) + pynutil.delete('"') + delete_space ) @@ -71,13 +85,25 @@ def __init__(self, decimal: GraphFst, cardinal: GraphFst): + pynutil.delete('"') ) optional_fractional = pynini.closure(fractional + delete_space, 0, 1) - graph = ( - (graph_cardinal | graph_decimal) + # Graph with no space for time units + graph_no_space = ( + pynini.union(graph_cardinal, graph_decimal) + + delete_space + + optional_fractional + + unit_no_space + + delete_space + ) + + # Graph with space for other units + graph_with_space = ( + pynini.union(graph_cardinal, graph_decimal) + delete_space + optional_fractional - + pynutil.insert(" ") - + unit + + insert_space + + unit_with_space + delete_space ) + + graph = pynini.union(graph_no_space, graph_with_space) delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/money.py index dbea81ff4..e4815d0d7 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/money.py @@ -14,9 +14,15 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_CHAR, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( + NEMO_CHAR, + NEMO_SPACE, + GraphFst, + delete_space, +) + class MoneyFst(GraphFst): """ @@ -33,7 +39,7 @@ def __init__(self, decimal: GraphFst): pynutil.delete("currency:") + delete_space + pynutil.delete('"') - + pynini.closure(NEMO_CHAR - " ", 1) + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + pynutil.delete('"') ) graph = decimal.numbers + delete_space + unit diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py index c45e1ee7b..8ada63998 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/ordinal.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py index d0e8de0b8..e65d1f018 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/telephone.py @@ -14,20 +14,43 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_space, + insert_space, +) + class TelephoneFst(GraphFst): """ Finite state transducer for verbalizing telephone, e.g. - telephone { number_part: "1231235678" } - -> 1231235678 + telephone { number_part: "123-123-5678" } -> 123-123-5678 + telephone { country_code: "+84" number_part: "936-555-449" } -> +84 936-555-449 + telephone { number_part: "192.168.0.1" } -> 192.168.0.1 + telephone { number_part: "1234 5678 9101 2345" } -> 1234 5678 9101 2345 + telephone { number_part: "x86" } -> x86 """ def __init__(self): super().__init__(name="telephone", kind="verbalize") number_part = pynutil.delete('number_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') - delete_tokens = self.delete_tokens(number_part) + + # Optional country code + country_code = ( + pynutil.delete('country_code: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + delete_space + + insert_space + ) + + optional_country_code = pynini.closure(country_code, 0, 1) + + graph = optional_country_code + number_part + + delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/time.py index 446160611..6bb04e2fd 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/time.py @@ -14,14 +14,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, + NEMO_SPACE, GraphFst, delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -36,7 +38,7 @@ class TimeFst(GraphFst): def __init__(self): super().__init__(name="time", kind="verbalize") - add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT) + add_leading_zero_to_double_digit = pynini.union(NEMO_DIGIT + NEMO_DIGIT, pynutil.insert("0") + NEMO_DIGIT) hour = ( pynutil.delete("hours:") + delete_space @@ -64,12 +66,14 @@ def __init__(self): + pynutil.delete("zone:") + delete_space + pynutil.delete('"') - + pynini.closure(NEMO_CHAR - " ", 1) + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + pynutil.delete('"') ) optional_zone = pynini.closure(zone, 0, 1) optional_second = pynini.closure( - delete_space + pynutil.insert(":") + (second @ add_leading_zero_to_double_digit), 0, 1, + delete_space + pynutil.insert(":") + (second @ add_leading_zero_to_double_digit), + 0, + 1, ) graph_h = hour + pynutil.insert("h") @@ -84,6 +88,6 @@ def __init__(self): + pynutil.insert("s") ) - graph = (graph_h | graph_ms | graph_hms) + optional_zone + graph = pynini.union(graph_h, graph_ms, graph_hms) + optional_zone delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py index 40899e191..9b982f965 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/verbalize_final.py @@ -14,10 +14,11 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_extra_space, delete_space from nemo_text_processing.inverse_text_normalization.vi.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.vi.verbalizers.word import WordFst -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py index 558c75451..1068a24a0 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/whitelist.py @@ -15,13 +15,15 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( NEMO_CHAR, NEMO_SIGMA, + NEMO_SPACE, GraphFst, delete_space, ) -from pynini.lib import pynutil class WhiteListFst(GraphFst): @@ -36,8 +38,8 @@ def __init__(self): pynutil.delete("name:") + delete_space + pynutil.delete('"') - + pynini.closure(NEMO_CHAR - " ", 1) + + pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + pynutil.delete('"') ) - graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", NEMO_SPACE), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/word.py index 3d7d6f8db..99c56a953 100644 --- a/nemo_text_processing/inverse_text_normalization/vi/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/vi/verbalizers/word.py @@ -14,13 +14,15 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.vi.graph_utils import ( NEMO_CHAR, NEMO_SIGMA, + NEMO_SPACE, GraphFst, delete_space, ) -from pynini.lib import pynutil class WordFst(GraphFst): @@ -31,8 +33,8 @@ class WordFst(GraphFst): def __init__(self): super().__init__(name="word", kind="verbalize") - chars = pynini.closure(NEMO_CHAR - " ", 1) + chars = pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) char = pynutil.delete("name:") + delete_space + pynutil.delete('"') + chars + pynutil.delete('"') - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", NEMO_SPACE), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/zh/graph_utils.py b/nemo_text_processing/inverse_text_normalization/zh/graph_utils.py index 13e8ab6d0..8d8217644 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/zh/graph_utils.py @@ -22,12 +22,14 @@ from pynini.export import export from pynini.lib import byte, pynutil, utf8 +from nemo_text_processing.inverse_text_normalization.zh.utils import load_labels + NEMO_CHAR = utf8.VALID_UTF8_CHAR NEMO_DIGIT = byte.DIGIT NEMO_HEX = pynini.union(*string.hexdigits).optimize() -NEMO_NON_BREAKING_SPACE = "\u00A0" +NEMO_NON_BREAKING_SPACE = "\u00a0" NEMO_SPACE = " " -NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize() +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() @@ -84,7 +86,10 @@ def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): written_capitalized = written[0].upper() + written[1:] additional_labels.extend( [ - [written_capitalized, spoken.capitalize(),], # first letter capitalized + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized [ written_capitalized, spoken.upper().replace(" AND ", " and "), @@ -98,7 +103,10 @@ def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): print(f"This is weight {weight}") if len(weight) == 0: additional_labels.extend( - [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()],] + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] ) else: additional_labels.extend( @@ -179,4 +187,4 @@ def delete_tokens(self, fst) -> "pynini.FstLike": + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/cardinal.py index b29fc5fb3..0715a3988 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst from nemo_text_processing.inverse_text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class CardinalFst(GraphFst): @@ -24,7 +25,7 @@ def __init__(self): Fitite state transducer for classifying cardinals (e.g., 负五十 -> cardinal { negative: "-" integer: "50" }) This class converts cardinals up to hundred millions (i.e., (10**10)) Single unit digits are not converted (e.g., 五 -> 五) - Numbers less than 20 are not converted. + Numbers less than 20 are not converted. 二十 (2 characters/logograms) is kept as it is but 二十一 (3 characters/logograms) would become 21 """ super().__init__(name="cardinal", kind="classify") @@ -78,7 +79,11 @@ def __init__(self): | (graph_digits + delete_ten_thousands + pynini.cross(pynini.closure("零"), "00") + graph_all) | (graph_digits + delete_ten_thousands + pynini.cross(pynini.closure("零"), "000") + graph_digits) ) - graph_ten_thousands = graph_ten_thousands_simple | graph_ten_thousands_complex | pynutil.insert("00000") + graph_ten_thousands = ( + pynutil.add_weight(graph_ten_thousands_simple, -1.0) + | graph_ten_thousands_complex + | pynutil.insert("00000") + ) # grammmar for hundred thousands 十万 graph_hundred_thousands_simple = graph_all + closure_ten_thousands @@ -88,8 +93,10 @@ def __init__(self): | (graph_all + delete_ten_thousands + pynini.cross(pynini.closure("零"), "00") + graph_all) | (graph_all + delete_ten_thousands + pynini.cross(pynini.closure("零"), "000") + graph_digits) ) - graph_hundred_thousands = (graph_hundred_thousands_simple | graph_hundred_thousands_complex) | pynutil.insert( - "000000" + graph_hundred_thousands = ( + pynutil.add_weight(graph_hundred_thousands_simple, -1.0) + | graph_hundred_thousands_complex + | pynutil.insert("000000") ) # grammar for millions 百万 @@ -103,7 +110,12 @@ def __init__(self): + graph_hundreds_complex ) | (graph_hundreds_complex + delete_ten_thousands + pynini.cross(pynini.closure("零"), "00") + graph_all) - | (graph_hundreds_complex + delete_ten_thousands + pynini.cross(pynini.closure("零"), "000") + graph_digits) + | ( + graph_hundreds_complex + + delete_ten_thousands + + pynini.cross(pynini.closure("零"), "000") + + graph_digits + ) ) graph_millions = ( pynutil.add_weight(graph_millions_simple, -1.0) | graph_millions_complex | pynutil.insert("0000000") @@ -168,7 +180,9 @@ def __init__(self): | (graph_digits + delete_hundred_millions + pynini.cross(pynini.closure("零"), "0000000") + graph_digits) ) graph_hundred_millions = ( - graph_hundred_millions_simple | graph_hundred_millions_complex | pynutil.insert("000000000") + pynutil.add_weight(graph_hundred_millions_simple, -1.0) + | graph_hundred_millions_complex + | pynutil.insert("000000000") ) # grammar for billions 十亿 @@ -203,7 +217,9 @@ def __init__(self): | (graph_all + delete_hundred_millions + pynini.cross(pynini.closure("零"), "000000") + graph_all) | (graph_all + delete_hundred_millions + pynini.cross(pynini.closure("零"), "0000000") + graph_digits) ) - graph_billions = graph_billions_simple | graph_billions_complex | pynutil.insert("0000000000") + graph_billions = ( + pynutil.add_weight(graph_billions_simple, -1.0) | graph_billions_complex | pynutil.insert("0000000000") + ) # grammar for ten billions 百亿 graph_ten_billions_simple = graph_hundreds_complex + closure_hundred_millions @@ -252,7 +268,11 @@ def __init__(self): + graph_digits ) ) - graph_ten_billions = graph_ten_billions_simple | graph_ten_billions_complex | pynutil.insert("00000000000") + graph_ten_billions = ( + pynutil.add_weight(graph_ten_billions_simple, -1.0) + | graph_ten_billions_complex + | pynutil.insert("00000000000") + ) # grammar for hundred billions 千亿 graph_hundred_billions_simple = graph_thousands_complex + closure_hundred_millions @@ -301,7 +321,9 @@ def __init__(self): + graph_digits ) ) - graph_hundred_billions = graph_hundred_billions_simple | graph_hundred_billions_complex + graph_hundred_billions = ( + pynutil.add_weight(graph_hundred_billions_simple, -1.0) | graph_hundred_billions_complex + ) # combining grammar; output for cardinal grammar graph = pynini.union( diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/date.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/date.py index 55e77aeba..108c222fd 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/date.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst from nemo_text_processing.inverse_text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class DateFst(GraphFst): @@ -60,7 +61,9 @@ def __init__(self): # graph_date = graph_year | graph_month | graph_day # grammar for optional prefix ad or bc - graph_bc_prefix = pynini.closure("紀元前", 0, 1) | pynini.closure("公元前", 0, 1) | pynini.closure("纪元前", 0, 1) + graph_bc_prefix = ( + pynini.closure("紀元前", 0, 1) | pynini.closure("公元前", 0, 1) | pynini.closure("纪元前", 0, 1) + ) graph_bc = pynutil.delete(graph_bc_prefix) graph_ad_prefix = ( diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/decimal.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/decimal.py index f334f2675..8b702a0fe 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/decimal.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst from nemo_text_processing.inverse_text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil def get_quantity(decimal, cardinal): @@ -68,35 +69,20 @@ class DecimalFst(GraphFst): def __init__(self, cardinal: GraphFst): super().__init__(name="decimal", kind="classify") - cardinal_after_decimal = pynini.string_file(get_abs_path("data/numbers/digit-nano.tsv")) - cardinal_before_decimal = cardinal.just_cardinals | (pynini.closure(pynini.cross("零", "0"), 0, 1)) - - delete_decimal = pynutil.delete("点") | pynutil.delete( - "點" - ) # delete decimal character, 'point' in english in 'one point two for 1.2' - - # grammar for integer part - graph_integer = ( - pynutil.insert('integer_part: "') - + (cardinal_before_decimal | (pynini.closure(pynini.cross("零", "0"), 0, 1))) - + pynutil.insert('" ') - ) # tokenization on just numbers - graph_integer_or_none = graph_integer | pynutil.insert('integer_part: "0" ', weight=0.01) # integer or zero - - # grammar for fractional part - delete_zero = pynini.closure(pynini.cross("零", "0")) - graph_string_of_cardinals = cardinal_after_decimal - graph_string_of_cardinals = pynini.closure( - (pynini.closure(graph_string_of_cardinals) + delete_zero + pynini.closure(graph_string_of_cardinals)), 1 - ) - graph_fractional = pynini.closure( - pynutil.insert('fractional_part: "') + graph_string_of_cardinals + pynutil.insert('"'), 1 + cardinal_after_decimal = pynini.string_file(get_abs_path("data/numbers/digit-nano.tsv")) | pynini.closure( + pynini.cross("零", "0") ) + cardinal_before_decimal = cardinal.just_cardinals | pynini.cross("零", "0") + + delete_decimal = pynutil.delete("点") | pynutil.delete("點") + + graph_integer = pynutil.insert('integer_part: "') + cardinal_before_decimal + pynutil.insert('" ') + + graph_string_of_cardinals = pynini.closure(cardinal_after_decimal, 1) + graph_fractional = pynutil.insert('fractional_part: "') + graph_string_of_cardinals + pynutil.insert('"') - # grammar for decimal: integer+delete character+part after decimal point - graph_decimal_no_sign = pynini.closure((graph_integer_or_none + delete_decimal + graph_fractional), 1) + graph_decimal_no_sign = pynini.closure((graph_integer + delete_decimal + graph_fractional), 1) - # New Grammar added for Money self.final_graph_wo_negative = graph_decimal_no_sign | get_quantity( graph_decimal_no_sign, cardinal.just_cardinals ) diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/fraction.py index 33fcd20a9..49fd428c1 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/fraction.py @@ -13,16 +13,17 @@ # limitations under the License. -from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst + class FractionFst(GraphFst): """ Finite state transducer for classifying fraction e.g. 二分之一 -> tokens { fraction { denominator: "2" numerator: "1"} } e.g. 五又二分之一 -> tokens { fraction { integer_part: "1" denominator: "2" numerator: "1" } } - + Args: cardinal: CardinalFst """ diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/money.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/money.py index d0a24ab3b..477a82f5d 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/money.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_DIGIT, GraphFst from nemo_text_processing.inverse_text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class MoneyFst(GraphFst): @@ -56,7 +57,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): # yuan major plus minor major_symbol = pynini.accep("块") | pynini.cross("塊", "块") - tencent = pynini.accep("毛") | pynini.accep("角",) + tencent = pynini.accep("毛") | pynini.accep( + "角", + ) cent = pynini.accep("分") graph_kuai = ( graph_integer_component diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/ordinal.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/ordinal.py index 47ffbdd36..3c3ba1fdc 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst + class OrdinalFst(GraphFst): def __init__(self, cardinal: GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/punctuation.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/punctuation.py index 4ca8eab9b..320f9272a 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/punctuation.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/punctuation.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst + class PunctuationFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/time.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/time.py index 9a3aca388..288043367 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/time.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/time.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst from nemo_text_processing.inverse_text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py index a46563170..3364ed4b2 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import ( GraphFst, delete_extra_space, delete_space, + delete_zero_or_one_space, generator_main, ) from nemo_text_processing.inverse_text_normalization.zh.taggers.cardinal import CardinalFst @@ -32,13 +34,13 @@ from nemo_text_processing.inverse_text_normalization.zh.taggers.time import TimeFst from nemo_text_processing.inverse_text_normalization.zh.taggers.whitelist import WhiteListFst from nemo_text_processing.inverse_text_normalization.zh.taggers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): """ Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. - For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. More details to deployment at NeMo/tools/text_processing_deployment. Args: @@ -47,7 +49,11 @@ class ClassifyFst(GraphFst): """ def __init__( - self, input_case: str, cache_dir: str = None, whitelist: str = None, overwrite_cache: bool = False, + self, + input_case: str, + cache_dir: str = None, + whitelist: str = None, + overwrite_cache: bool = False, ): super().__init__(name="tokenize_and_classify", kind="classify") @@ -57,9 +63,9 @@ def __init__( far_file = os.path.join(cache_dir, "_zh_itn.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") cardinal = CardinalFst() cardinal_graph = cardinal.fst @@ -91,16 +97,15 @@ def __init__( ) punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") - token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") token_plus_punct = ( pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) ) - graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = token_plus_punct + pynini.closure(delete_zero_or_one_space + token_plus_punct) graph = delete_space + graph + delete_space self.fst = graph.optimize() if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/whitelist.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/whitelist.py index 8e0cbd328..c8ed1c2a3 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/whitelist.py @@ -15,9 +15,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import INPUT_LOWER_CASED, GraphFst from nemo_text_processing.inverse_text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/zh/taggers/word.py b/nemo_text_processing/inverse_text_normalization/zh/taggers/word.py index 3e129fb98..6f0411b64 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/zh/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ @@ -25,5 +26,5 @@ class WordFst(GraphFst): def __init__(self): super().__init__(name="word", kind="classify") - word = pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"') + word = pynutil.insert('name: "') + NEMO_NOT_SPACE + pynutil.insert('"') self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/zh/utils.py b/nemo_text_processing/inverse_text_normalization/zh/utils.py index d63a1b2f7..8db669ff6 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/utils.py +++ b/nemo_text_processing/inverse_text_normalization/zh/utils.py @@ -27,7 +27,7 @@ def num_to_word(x: Union[str, int]): Args x: integer - Returns: spoken representation + Returns: spoken representation """ if isinstance(x, int): x = str(x) @@ -41,7 +41,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path @@ -60,3 +60,17 @@ def get_various_formats(text: str) -> List[str]: result.append(t.upper()) result.append(t.capitalize()) return result + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py index 3eec1a88b..f33987173 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/cardinal.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, GraphFst, delete_space, ) -from pynini.lib import pynutil class CardinalFst(GraphFst): @@ -27,7 +28,7 @@ def __init__(self): super().__init__(name="cardinal", kind="verbalize") # group numbers by three - exactly_three_digits = NEMO_DIGIT ** 3 + exactly_three_digits = NEMO_DIGIT**3 at_most_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) suffix = pynini.union( diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/date.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/date.py index 2b979e6b8..60606aea0 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/date.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DateFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py index ea8fa4ab0..b36e44dfa 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/decimal.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, GraphFst, delete_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): @@ -27,19 +28,11 @@ def __init__(self): super().__init__(name="decimal", kind="verbalize") # group numbers by three - exactly_three_digits = NEMO_DIGIT ** 3 + exactly_three_digits = NEMO_DIGIT**3 at_most_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # insert a "," for every three numbers before decimal point space_every_three_integer = at_most_three_digits + (pynutil.insert(",") + exactly_three_digits).closure() - # insert a "," for every three numbers after decimal point - space_every_three_decimal = ( - pynini.accep(".") + (exactly_three_digits + pynutil.insert(",")).closure() + at_most_three_digits - ) - - # combine both - group_by_threes = space_every_three_integer | space_every_three_decimal - self.group_by_threes = group_by_threes # removing tokenizations, 'negative: ' optional_sign = pynini.closure( @@ -56,10 +49,10 @@ def __init__(self): pynutil.delete("integer_part:") + delete_space + pynutil.delete('"') - + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete('"') ) - integer = integer @ group_by_threes + integer = integer @ space_every_three_integer optional_integer = pynini.closure(integer + delete_space, 0, 1) # removing tokenizations, 'fractionl_part' @@ -81,10 +74,11 @@ def __init__(self): + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ) - optional_quantity = pynini.closure(quantity + delete_space) + optional_quantity = pynini.closure(delete_space + quantity) # combining graphs removing tokenizations *3 graph = (optional_integer + optional_fractional + optional_quantity).optimize() + graph = optional_sign + graph # add optional sign for negative number self.numebrs = graph delete_tokens = self.delete_tokens(graph) diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py index d5ea2ced1..57dcbd95f 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/fraction.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_DIGIT, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_DIGIT, GraphFst, delete_space + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/money.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/money.py index 2fd3919a4..92ba6d90d 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/money.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/money.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, GraphFst, delete_space, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/ordinal.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/ordinal.py index 93f2a678d..13731724e 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/ordinal.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/ordinal.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, GraphFst, delete_space, ) -from pynini.lib import pynutil class OrdinalFst(GraphFst): diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/time.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/time.py index 4560fdf62..f0ea1bd28 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/time.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/time.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_DIGIT, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.inverse_text_normalization.zh.graph_utils import NEMO_DIGIT, GraphFst, delete_space + class TimeFst(GraphFst): """ diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py index b379c4d94..5368e2c42 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize.py @@ -26,7 +26,7 @@ class VerbalizeFst(GraphFst): """ Composes other verbalizer grammars. - For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. More details to deployment at NeMo/tools/text_processing_deployment. """ diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py index e21b1d332..d89e7bc50 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py @@ -13,15 +13,16 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst, delete_extra_space, delete_space from nemo_text_processing.inverse_text_normalization.zh.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.zh.verbalizers.word import WordFst -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now """ @@ -39,5 +40,5 @@ def __init__(self): + delete_space + pynutil.delete("}") ) - graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + graph = delete_space + pynini.closure(graph + delete_space) + graph + delete_space self.fst = graph diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py index 994935b2b..571070a2e 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/whitelist.py @@ -14,13 +14,14 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import ( NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space, ) -from pynini.lib import pynutil class WhiteListFst(GraphFst): @@ -38,5 +39,5 @@ def __init__(self): + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete('"') ) - graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/word.py index 5888e2d8c..c3e76fef7 100644 --- a/nemo_text_processing/inverse_text_normalization/zh/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/zh/verbalizers/word.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.inverse_text_normalization.zh.graph_utils import ( NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space, ) -from pynini.lib import pynutil class WordFst(GraphFst): @@ -32,6 +33,6 @@ def __init__(self): super().__init__(name="word", kind="verbalize") chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete('"') + chars + pynutil.delete('"') - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/package_info.py b/nemo_text_processing/package_info.py index 67ef65e94..face7d049 100644 --- a/nemo_text_processing/package_info.py +++ b/nemo_text_processing/package_info.py @@ -13,10 +13,10 @@ # limitations under the License. -MAJOR = 0 -MINOR = 2 +MAJOR = 1 +MINOR = 1 PATCH = 0 -PRE_RELEASE = 'rc0' +PRE_RELEASE = '' # Use the following formatting: (major, minor, patch, pre-release) VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE) diff --git a/nemo_text_processing/text_normalization/ar/__init__.py b/nemo_text_processing/text_normalization/ar/__init__.py index 0e2a7e217..3c3c1ac96 100644 --- a/nemo_text_processing/text_normalization/ar/__init__.py +++ b/nemo_text_processing/text_normalization/ar/__init__.py @@ -12,18 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - from nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.utils.logging import logger try: import pynini PYNINI_AVAILABLE = True except (ModuleNotFoundError, ImportError): - logging.warning( + logger.warning( "`pynini` is not installed ! \n" "Please run the `nemo_text_processing/setup.sh` script" "prior to usage of this toolkit." diff --git a/nemo_text_processing/text_normalization/ar/graph_utils.py b/nemo_text_processing/text_normalization/ar/graph_utils.py index 067448ac6..54685a093 100644 --- a/nemo_text_processing/text_normalization/ar/graph_utils.py +++ b/nemo_text_processing/text_normalization/ar/graph_utils.py @@ -13,19 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import string from pathlib import Path from typing import Dict from nemo_text_processing.text_normalization.en.utils import get_abs_path +from nemo_text_processing.utils.logging import logger try: import pynini from pynini import Far - from pynini.export import export from pynini.examples import plurals + from pynini.export import export from pynini.lib import byte, pynutil, utf8 NEMO_CHAR = utf8.VALID_UTF8_CHAR @@ -36,9 +36,9 @@ NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() NEMO_HEX = pynini.union(*string.hexdigits).optimize() - NEMO_NON_BREAKING_SPACE = u"\u00A0" + NEMO_NON_BREAKING_SPACE = u"\u00a0" NEMO_SPACE = " " - NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00A0").optimize() + NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() @@ -163,7 +163,7 @@ NEMO_ALPHA = None NEMO_ALNUM = None NEMO_HEX = None - NEMO_NON_BREAKING_SPACE = u"\u00A0" + NEMO_NON_BREAKING_SPACE = u"\u00a0" NEMO_SPACE = " " NEMO_WHITE_SPACE = None NEMO_NOT_SPACE = None @@ -210,7 +210,7 @@ def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): for rule, graph in graphs.items(): exporter[rule] = graph.optimize() exporter.close() - logging.info(f'Created {file_name}') + logger.info(f'Created {file_name}') def get_plurals(fst): @@ -317,4 +317,4 @@ def delete_tokens(self, fst) -> 'pynini.FstLike': + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/ar/taggers/__init__.py b/nemo_text_processing/text_normalization/ar/taggers/__init__.py index 1f7941250..8944d9e6c 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/__init__.py +++ b/nemo_text_processing/text_normalization/ar/taggers/__init__.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - import pynini + from nemo_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.utils.logging import logger diff --git a/nemo_text_processing/text_normalization/ar/taggers/cardinal.py b/nemo_text_processing/text_normalization/ar/taggers/cardinal.py index 1ac137e78..a6ab7aca3 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/ar/taggers/cardinal.py @@ -13,14 +13,15 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst, flop_digits, insert_and, insert_space from nemo_text_processing.text_normalization.ar.utils import get_abs_path -from pynini.lib import pynutil class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. "9837" -> cardinal { integer: "تسعة اَلاف وثمان مئة وسبعة وثلاثون" } Args: diff --git a/nemo_text_processing/text_normalization/ar/taggers/decimal.py b/nemo_text_processing/text_normalization/ar/taggers/decimal.py index 9fd134399..72d2dc47b 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/ar/taggers/decimal.py @@ -13,40 +13,16 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, insert_space -from nemo_text_processing.text_normalization.ar.utils import get_abs_path from pynini.lib import pynutil - -def get_quantity(decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike") -> "pynini.FstLike": - """ - Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, - e.g. 5 مليون -> integer_part: "خمسة" quantity: "مليون" - e.g. 5.4 مليون -> integer_part: "خمسة" fractional_part: "اربعة من عشرة" quantity: "مليون" - - Args: - decimal: decimal FST - cardinal_up_to_hundred: cardinal FST - """ - numbers = cardinal_up_to_hundred - - res = ( - pynutil.insert('integer_part: "') - + numbers - + pynutil.insert('"') - + pynini.accep(" ") - + pynutil.insert('quantity: "') - + quantities - + pynutil.insert('"') - ) - res |= decimal + pynini.accep(" ") + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') - return res +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, insert_space +from nemo_text_processing.text_normalization.ar.utils import get_abs_path class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. - 321.7 --> ثلاث مئة وواحد وعشرون وسبعة من عشرة + Finite state transducer for classifying decimal, e.g. + 321.7 --> ثلاث مئة وواحد وعشرون وسبعة من عشرة -321.7 -> decimal { negative: "true" integer_part: "321" fractional_part: ".7" } cardinal: CardinalFst """ diff --git a/nemo_text_processing/text_normalization/ar/taggers/fraction.py b/nemo_text_processing/text_normalization/ar/taggers/fraction.py index 74e568588..1ef390506 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/ar/taggers/fraction.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst from nemo_text_processing.text_normalization.ar.utils import get_abs_path -from pynini.lib import pynutil class FractionFst(GraphFst): @@ -25,7 +26,7 @@ class FractionFst(GraphFst): tokens { fraction { integer_part: "واحد" numerator: "واحد" denominator: "نص" } } Args: - cardinal: cardinal fst + cardinal: cardinal fst """ def __init__(self, cardinal): diff --git a/nemo_text_processing/text_normalization/ar/taggers/measure.py b/nemo_text_processing/text_normalization/ar/taggers/measure.py index 2aca51201..ce22f3d76 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/measure.py +++ b/nemo_text_processing/text_normalization/ar/taggers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -22,7 +24,6 @@ insert_space, ) from nemo_text_processing.text_normalization.ar.utils import get_abs_path -from pynini.lib import pynutil unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) @@ -54,7 +55,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) optional_unit_denominator = pynini.closure( - pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, 0, 1, + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, ) unit_plural = ( @@ -75,15 +78,14 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) subgraph_cardinal = ( - (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst - + insert_space - + pynini.closure(pynutil.delete(" "), 0, 1) - + unit_plural - | unit_plural - + pynini.closure(pynutil.delete(" "), 0, 1) - + insert_space - + (optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1")) @ cardinal.fst - ) + optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1") + ) @ cardinal.fst + insert_space + pynini.closure( + pynutil.delete(" "), 0, 1 + ) + unit_plural | unit_plural + pynini.closure( + pynutil.delete(" "), 0, 1 + ) + insert_space + ( + optional_graph_negative + (pynini.closure(NEMO_DIGIT) - "1") + ) @ cardinal.fst subgraph_cardinal |= ( (optional_graph_negative + pynini.accep("1")) @ cardinal.fst diff --git a/nemo_text_processing/text_normalization/ar/taggers/money.py b/nemo_text_processing/text_normalization/ar/taggers/money.py index 266edf652..b809354e4 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/money.py +++ b/nemo_text_processing/text_normalization/ar/taggers/money.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -21,7 +23,6 @@ insert_space, ) from nemo_text_processing.text_normalization.ar.utils import get_abs_path, load_labels -from pynini.lib import pynutil min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) @@ -35,7 +36,7 @@ class MoneyFst(GraphFst): "$1,99" -> money { integer_part: "سبعة" currency_maj: "دولار" fractional_part: "تسعة وتسعون" currency_min: "سنت" preserve_order: true} "$0,10" -> money { fractional_part: "عشرة" currency_min: "بنسات" preserve_order: true } "$9" -> money { integer_part: "تسعة" currency_maj: "دولار" preserve_order: true} - + Args: cardinal: CardinalFst deterministic: if True will provide a single transduction option, @@ -79,14 +80,14 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"") ) - graph_integer_only = graph_maj_singular + insert_space + graph_integer_one - graph_integer_only |= graph_maj_plural + insert_space + graph_integer + currency_first = pynutil.insert(' morphosyntactic_features: "currency_first"') + # Currency-first tagging for exactly one major unit (e.g. $1 -> دولار واحد). + graph_integer_one_unit = graph_maj_singular + insert_space + graph_integer_one + currency_first # For local currency "9د.ك" graph_integer_only_ar = graph_integer + insert_space + graph_ar_cur - # graph_decimal_ar = graph_decimal_final + insert_space + graph_ar_cur - graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_integer_only_ar + graph = (graph_integer_one_unit + optional_delete_fractional_zeros) | graph_integer_only_ar # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 @@ -111,9 +112,12 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): preserve_order = pynutil.insert(" preserve_order: true") integer_plus_maj = graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural - integer_plus_maj |= graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular - # non zero integer part - integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + integer_plus_maj_with_one = integer_plus_maj | ( + graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular + ) + # Amount == 1 without fractional part uses graph_integer_one_unit / graph_one_prefix. + integer_plus_maj_no_minor = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + integer_plus_maj_with_minor = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj_with_one graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "") graph_fractional_one = pynutil.insert("fractional_part: \"") + graph_fractional_one + pynutil.insert("\"") @@ -140,19 +144,16 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): graph_fractional_up_to_ten + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural ) - graph_with_no_minor_curr = integer_plus_maj - graph_with_no_minor_curr |= pynutil.add_weight(integer_plus_maj, weight=0.0001,) - - graph_with_no_minor_curr = pynutil.delete(curr_symbol) + graph_with_no_minor_curr + preserve_order + graph_with_no_minor_curr = pynutil.delete(curr_symbol) + integer_plus_maj_no_minor + preserve_order graph_with_no_minor = ( graph_with_no_minor_curr if graph_with_no_minor is None else pynini.union(graph_with_no_minor, graph_with_no_minor_curr) ) - decimal_graph_with_minor_curr = integer_plus_maj + pynini.cross(".", " ") + fractional_plus_min + decimal_graph_with_minor_curr = integer_plus_maj_with_minor + pynini.cross(".", " ") + fractional_plus_min decimal_graph_with_minor_curr |= pynutil.add_weight( - integer_plus_maj + integer_plus_maj_with_minor + pynini.cross(".", " ") + pynutil.insert("fractional_part: \"") + two_digits_fractional_part @ cardinal_graph diff --git a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py index 833652d76..5f3155ae2 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ar/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -31,7 +32,7 @@ from nemo_text_processing.text_normalization.ar.taggers.money import MoneyFst from nemo_text_processing.text_normalization.ar.taggers.word import WordFst from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -69,9 +70,9 @@ def __init__( self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") self.cardinal = CardinalFst() cardinal_graph = self.cardinal.fst @@ -113,4 +114,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ar/taggers/word.py b/nemo_text_processing/text_normalization/ar/taggers/word.py index facd67bce..26a413028 100644 --- a/nemo_text_processing/text_normalization/ar/taggers/word.py +++ b/nemo_text_processing/text_normalization/ar/taggers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ar/utils.py b/nemo_text_processing/text_normalization/ar/utils.py index 054703e84..1ad8f9927 100644 --- a/nemo_text_processing/text_normalization/ar/utils.py +++ b/nemo_text_processing/text_normalization/ar/utils.py @@ -22,7 +22,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path @@ -37,14 +37,14 @@ def load_labels(abs_path): Returns dictionary of mappings """ - label_tsv = open(abs_path, encoding="utf-8") - labels = list(csv.reader(label_tsv, delimiter="\t")) + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) return labels def augment_labels_with_punct_at_end(labels): """ - augments labels: if key ends on a punctuation that value does not have, add a new label + augments labels: if key ends on a punctuation that value does not have, add a new label where the value maintains the punctuation Args: diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/__init__.py b/nemo_text_processing/text_normalization/ar/verbalizers/__init__.py index 1f7941250..8944d9e6c 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/__init__.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/__init__.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - import pynini + from nemo_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.utils.logging import logger diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/ar/verbalizers/cardinal.py index 2125e9a65..ec3a51838 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/decimal.py b/nemo_text_processing/text_normalization/ar/verbalizers/decimal.py index 28707f872..2903f435c 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py index 1b3330cec..3ce7e3ddd 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/fraction.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py index e2f3d19f8..b762eaa3b 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/measure.py @@ -13,20 +13,21 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_preserve_order, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): """ Finite state transducer for verbalizing measure, e.g. measure { cardinal { integer: "20" } units: "%" } -> "عشرون في المائة" - + Args: decimal: decimal GraphFst cardinal: cardinal GraphFst diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/money.py b/nemo_text_processing/text_normalization/ar/verbalizers/money.py index 57883bb52..9f5041b13 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/money.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_preserve_order, delete_space, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): @@ -27,6 +28,7 @@ class MoneyFst(GraphFst): Finite state transducer for verbalizing money, e.g. money { integer_part: "تسعة" currency_maj: "يورو" preserve_order: true} -> "تسعة يورو" money { integer_part: "تسعة" currency_maj: "دولار" preserve_order: true} -> "تسعة دولار" + money { currency_maj: "دولار" integer_part: "واحد" morphosyntactic_features: "currency_first"} -> "دولار واحد" money { integer_part: "خمسة" currency_maj: "دينار كويتي"} -> "خمسة دينار كويتي" Args: @@ -48,9 +50,10 @@ def __init__(self, deterministic: bool = True): integer_part = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") add_and = pynutil.insert(" و") + morph_currency_first = pynutil.delete(' morphosyntactic_features: "currency_first"') - # *** currency_maj - graph_integer = maj + keep_space + integer_part + # currency_maj before integer_part; disambiguated via morphosyntactic_features for Sparrowhawk. + graph_currency_first = maj + keep_space + integer_part + delete_space + morph_currency_first # *** currency_maj + (***) (و) *** current_min graph_integer_with_minor = ( @@ -60,17 +63,14 @@ def __init__(self, deterministic: bool = True): + delete_space + add_and + fractional_part - + keep_space + delete_space - + min + + pynini.closure(keep_space + min, 0, 1) + delete_preserve_order ) - # this graph fix word order from dollar three (دولار تسعة)--> three dollar (تسعة دولار) graph_integer_no_minor = integer_part + keep_space + maj + delete_space + delete_preserve_order - # *** current_min graph_minor = fractional_part + keep_space + delete_space + min + delete_preserve_order - graph = graph_integer | graph_integer_with_minor | graph_minor | graph_integer_no_minor + graph = graph_currency_first | graph_integer_with_minor | graph_minor | graph_integer_no_minor delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize_final.py index 282c97142..4145c2330 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.ar.graph_utils import ( GraphFst, delete_extra_space, @@ -24,13 +25,13 @@ ) from nemo_text_processing.text_normalization.ar.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.text_normalization.ar.verbalizers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): """ Finite state transducer that verbalizes an entire sentence - + Args: deterministic: if True will provide a single transduction option, for False multiple options (used for audio-based normalization) @@ -47,7 +48,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"ar_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst word = WordFst(deterministic=deterministic).fst @@ -67,4 +68,3 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ar/verbalizers/word.py b/nemo_text_processing/text_normalization/ar/verbalizers/word.py index 17ec7a310..2987d4ad1 100644 --- a/nemo_text_processing/text_normalization/ar/verbalizers/word.py +++ b/nemo_text_processing/text_normalization/ar/verbalizers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WordFst(GraphFst): """ @@ -31,6 +32,6 @@ def __init__(self, deterministic: bool = True): super().__init__(name="word", kind="verbalize", deterministic=deterministic) chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/data_loader_utils.py b/nemo_text_processing/text_normalization/data_loader_utils.py index f4cbaa28b..8faa0ffba 100644 --- a/nemo_text_processing/text_normalization/data_loader_utils.py +++ b/nemo_text_processing/text_normalization/data_loader_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,17 @@ import json -import logging import re import string import sys +import unicodedata from collections import defaultdict, namedtuple from typing import Dict, List, Optional, Set, Tuple from unicodedata import category +from nemo_text_processing.utils.logging import logger + +NFC = 'NFC' EOS_TYPE = "EOS" PUNCT_TYPE = "PUNCT" PLAIN_TYPE = "PLAIN" @@ -42,14 +45,16 @@ "FRACTION", "TIME", "ADDRESS", + "ROMAN", + "RANGE", ] -def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]: +def _load_kaggle_text_norm_file(file_path: str, to_lower: bool) -> List[Instance]: """ https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish Loads text file in the Kaggle Google text normalization file format: \t\t<`self` if trivial class or normalized text> - E.g. + E.g. PLAIN Brillantaisia PLAIN is PLAIN a @@ -65,7 +70,7 @@ def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]: Args: file_path: file path to text file - Returns: flat list of instances + Returns: flat list of instances """ res = [] with open(file_path, 'r') as fp: @@ -75,8 +80,9 @@ def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]: res.append(Instance(token_type=EOS_TYPE, un_normalized="", normalized="")) else: l_type, l_token, l_normalized = parts - l_token = l_token.lower() - l_normalized = l_normalized.lower() + if to_lower: + l_token = l_token.lower() + l_normalized = l_normalized.lower() if l_type == PLAIN_TYPE: res.append(Instance(token_type=l_type, un_normalized=l_token, normalized=l_token)) @@ -85,11 +91,11 @@ def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]: return res -def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file) -> List[Instance]: +def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file, to_lower: bool = True) -> List[Instance]: """ Load given list of text files using the `load_func` function. - Args: + Args: file_paths: list of file paths load_func: loading function @@ -97,7 +103,7 @@ def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file) -> """ res = [] for file_path in file_paths: - res.extend(load_func(file_path=file_path)) + res.extend(load_func(file_path=file_path, to_lower=to_lower)) return res @@ -117,7 +123,7 @@ def clean_generic(text: str) -> str: def evaluate(preds: List[str], labels: List[str], input: Optional[List[str]] = None, verbose: bool = True) -> float: """ - Evaluates accuracy given predictions and labels. + Evaluates accuracy given predictions and labels. Args: preds: predictions @@ -136,7 +142,7 @@ def evaluate(preds: List[str], labels: List[str], input: Optional[List[str]] = N acc = acc + 1 else: if input: - print(f"inpu: {json.dumps(input[i])}") + print(f"input: {json.dumps(input[i])}") print(f"gold: {json.dumps(label_norm)}") print(f"pred: {json.dumps(pred_norm)}") return acc / nums @@ -158,8 +164,8 @@ def training_data_to_tokens( for instance in data: if instance.token_type != EOS_TYPE: if category is None or instance.token_type == category: - result[instance.token_type][0].append(instance.un_normalized) - result[instance.token_type][1].append(instance.normalized) + result[instance.token_type][0].append(unicodedata.normalize(NFC, instance.un_normalized)) + result[instance.token_type][1].append(unicodedata.normalize(NFC, instance.normalized)) return result @@ -185,8 +191,13 @@ def training_data_to_sentences(data: List[Instance]) -> Tuple[List[str], List[st else: sentence.append(instance) sentence_categories.update([instance.token_type]) - un_normalized = [" ".join([instance.un_normalized for instance in sentence]) for sentence in sentences] - normalized = [" ".join([instance.normalized for instance in sentence]) for sentence in sentences] + un_normalized = [ + " ".join([unicodedata.normalize(NFC, instance.un_normalized) for instance in sentence]) + for sentence in sentences + ] + normalized = [ + " ".join([unicodedata.normalize(NFC, instance.normalized) for instance in sentence]) for sentence in sentences + ] return un_normalized, normalized, categories @@ -248,7 +259,7 @@ def load_file(file_path: str) -> List[str]: """ Loads given text file with separate lines into list of string. - Args: + Args: file_path: file path Returns: flat list of string @@ -267,7 +278,7 @@ def write_file(file_path: str, data: List[str]): Args: file_path: file path data: list of string - + """ with open(file_path, 'w') as fp: for line in data: @@ -344,7 +355,7 @@ def _is_valid(idx_out, idx_in, normalized_text, input): idx_out += 1 idx_in += 1 except: - logging.info(f"Skipping post-processing of {''.join(normalized_text)} for '{punct}'") + logger.info(f"Skipping post-processing of {''.join(normalized_text)} for '{punct}'") normalized_text = "".join(normalized_text) return re.sub(r' +', ' ', normalized_text) diff --git a/nemo_text_processing/text_normalization/de/data/electronic/domain.tsv b/nemo_text_processing/text_normalization/de/data/electronic/domain.tsv index 5738cd66c..771fc5049 100644 --- a/nemo_text_processing/text_normalization/de/data/electronic/domain.tsv +++ b/nemo_text_processing/text_normalization/de/data/electronic/domain.tsv @@ -1,9 +1,844 @@ .com punkt com +.net punkt net +.org punkt org +.jp punkt jp +.de punkt de .uk punkt uk .fr punkt fr -.net dot net .br punkt br -.in punkt in +.it punkt it .ru punkt ru -.de punkt de -.it punkt it \ No newline at end of file +.es punkt es +.me punkt me +.gov punkt gov +.pl punkt pl +.ca punkt ca +.au punkt au +.cn punkt cn +.co punkt co +.in punkt in +.nl punkt nl +.edu punkt edu +.info punkt info +.eu punkt eu +.ch punkt ch +.id punkt id +.at punkt at +.kr punkt kr +.cz punkt cz +.mx punkt mx +.be punkt be +.tv punkt tv +.se punkt se +.tr punkt tr +.tw punkt tw +.al punkt al +.ua punkt ua +.ir punkt ir +.vn punkt vn +.cl punkt cl +.sk punkt sk +.ly punkt ly +.cc punkt cc +.to punkt to +.no punkt no +.fi punkt fi +.us punkt us +.pt punkt pt +.dk punkt dk +.ar punkt ar +.hu punkt hu +.tk punkt tk +.gr punkt gr +.il punkt il +.news punkt news +.ro punkt ro +.my punkt my +.biz punkt biz +.ie punkt ie +.za punkt za +.nz punkt nz +.sg punkt sg +.ee punkt ee +.th punkt th +.io punkt io +.xyz punkt xyz +.pe punkt pe +.bg punkt bg +.hk punkt hk +.rs punkt rs +.lt punkt lt +.link punkt link +.ph punkt ph +.club punkt club +.si punkt si +.site punkt site +.mobi punkt mobi +.by punkt by +.cat punkt cat +.wiki punkt wiki +.la punkt la +.ga punkt ga +.xxx punkt xxx +.cf punkt cf +.hr punkt hr +.ng punkt ng +.jobs punkt jobs +.online punkt online +.kz punkt kz +.ug punkt ug +.gq punkt gq +.ae punkt ae +.is punkt is +.lv punkt lv +.pro punkt pro +.fm punkt fm +.tips punkt tips +.ms punkt ms +.sa punkt sa +.app punkt app +.lat punkt lat +.pk punkt pk +.ws punkt ws +.top punkt top +.xn--p1ai punkt xn--p1ai +.pw punkt pw +.ai punkt ai +.kw punkt kw +.ml punkt ml +.su punkt su +.lu punkt lu +.nu punkt nu +.ec punkt ec +.uy punkt uy +.az punkt az +.ma punkt ma +.st punkt st +.asia punkt asia +.im punkt im +.am punkt am +.email punkt email +.ke punkt ke +.eg punkt eg +.live punkt live +.md punkt md +.uz punkt uz +.today punkt today +.ge punkt ge +.ba punkt ba +.bo punkt bo +.blog punkt blog +.lk punkt lk +.do punkt do +.one punkt one +.ve punkt ve +.media punkt media +.sh punkt sh +.vip punkt vip +.life punkt life +.guru punkt guru +.mp punkt mp +.int punkt int +.tz punkt tz +.desi punkt desi +.jo punkt jo +.mk punkt mk +.np punkt np +.py punkt py +.mu punkt mu +.mn punkt mn +.re punkt re +.travel punkt travel +.tn punkt tn +.gd punkt gd +.sn punkt sn +.cam punkt cam +.shop punkt shop +.mil punkt mil +.tokyo punkt tokyo +.gg punkt gg +.kg punkt kg +.cr punkt cr +.li punkt li +.cu punkt cu +.hn punkt hn +.eus punkt eus +.rocks punkt rocks +.bd punkt bd +.dev punkt dev +.fun punkt fun +.video punkt video +.gt punkt gt +.bz punkt bz +.space punkt space +.store punkt store +.mt punkt mt +.name punkt name +.work punkt work +.porn punkt porn +.pub punkt pub +.kh punkt kh +.icu punkt icu +.as punkt as +.cy punkt cy +.pr punkt pr +.careers punkt careers +.ps punkt ps +.dz punkt dz +.iq punkt iq +.ltd punkt ltd +.pa punkt pa +.sv punkt sv +.qa punkt qa +.design punkt design +.sc punkt sc +.sy punkt sy +.aero punkt aero +.tech punkt tech +.best punkt best +.scot punkt scot +.win punkt win +.ni punkt ni +.bzh punkt bzh +.lb punkt lb +.city punkt city +.digital punkt digital +.red punkt red +.pg punkt pg +.world punkt world +.ag punkt ag +.ci punkt ci +.report punkt report +.ac punkt ac +.center punkt center +.ad punkt ad +.ovh punkt ovh +.om punkt om +.cloud punkt cloud +.press punkt press +.zw punkt zw +.vg punkt vg +.cx punkt cx +.website punkt website +.plus punkt plus +.page punkt page +.bs punkt bs +.cd punkt cd +.gh punkt gh +.moe punkt moe +.help punkt help +.download punkt download +.ao punkt ao +.cm punkt cm +.af punkt af +.click punkt click +.mg punkt mg +.tm punkt tm +.na punkt na +.pics punkt pics +.mv punkt mv +.market punkt market +.rw punkt rw +.fo punkt fo +.fj punkt fj +.tl punkt tl +.gratis punkt gratis +.art punkt art +.mc punkt mc +.mo punkt mo +.gal punkt gal +.coop punkt coop +.nc punkt nc +.run punkt run +.mz punkt mz +.tj punkt tj +.zone punkt zone +.cv punkt cv +.bh punkt bh +.place punkt place +.gy punkt gy +.agency punkt agency +.kn punkt kn +.mm punkt mm +.bid punkt bid +.bj punkt bj +.je punkt je +.gs punkt gs +.global punkt global +.lol punkt lol +.tube punkt tube +.lc punkt lc +.bm punkt bm +.ninja punkt ninja +.bn punkt bn +.review punkt review +.bi punkt bi +.ht punkt ht +.sm punkt sm +.ax punkt ax +.pm punkt pm +.ink punkt ink +.vc punkt vc +.so punkt so +.tel punkt tel +.cw punkt cw +.uno punkt uno +.casa punkt casa +.network punkt network +.buzz punkt buzz +.mw punkt mw +.vu punkt vu +.guide punkt guide +.bf punkt bf +.mq punkt mq +.sex punkt sex +.sr punkt sr +.zm punkt zm +.onl punkt onl +.vet punkt vet +.bw punkt bw +.jm punkt jm +.bar punkt bar +.gp punkt gp +.pf punkt pf +.ne punkt ne +.va punkt va +.bt punkt bt +.sd punkt sd +.expert punkt expert +.kim punkt kim +.africa punkt africa +.mr punkt mr +.gi punkt gi +.tg punkt tg +.cg punkt cg +.ls punkt ls +.sb punkt sb +.ck punkt ck +.gf punkt gf +.yt punkt yt +.dm punkt dm +.realtor punkt realtor +.aw punkt aw +.sz punkt sz +.london punkt london +.et punkt et +.ky punkt ky +.cool punkt cool +.lr punkt lr +.bb punkt bb +.support punkt support +.taipei punkt taipei +.sl punkt sl +.company punkt company +.dj punkt dj +.social punkt social +.gm punkt gm +.xn--mgberp4a5d4ar punkt xn--mgberp4a5d4ar +.love punkt love +.cymru punkt cymru +.directory punkt directory +.games punkt games +.bio punkt bio +.wang punkt wang +.watch punkt watch +.cafe punkt cafe +.care punkt care +.supply punkt supply +.ooo punkt ooo +.group punkt group +.tools punkt tools +.sx punkt sx +.rest punkt rest +.law punkt law +.wales punkt wales +.movie punkt movie +.monster punkt monster +.studio punkt studio +.wtf punkt wtf +.direct punkt direct +.education punkt education +.ye punkt ye +.fit punkt fit +.events punkt events +.berlin punkt berlin +.services punkt services +.church punkt church +.pn punkt pn +.tc punkt tc +.tf punkt tf +.nyc punkt nyc +.town punkt town +.xn--90ais punkt xn--90ais +.xn--j1amh punkt xn--j1amh +.ist punkt ist +.bet punkt bet +.sale punkt sale +.sport punkt sport +.brussels punkt brussels +.moscow punkt moscow +.menu punkt menu +.express punkt express +.vlaanderen punkt vlaanderen +.blue punkt blue +.cars punkt cars +.deals punkt deals +.host punkt host +.academy punkt academy +.training punkt training +.vi punkt vi +.men punkt men +.team punkt team +.canon punkt canon +.leclerc punkt leclerc +.solutions punkt solutions +.community punkt community +.land punkt land +.basketball punkt basketball +.gallery punkt gallery +.sexy punkt sexy +.codes punkt codes +.swiss punkt swiss +.camp punkt camp +.software punkt software +.auction punkt auction +.nrw punkt nrw +.style punkt style +.coffee punkt coffee +.paris punkt paris +.graphics punkt graphics +.holiday punkt holiday +.wine punkt wine +.dog punkt dog +.dating punkt dating +.radio punkt radio +.pet punkt pet +.parts punkt parts +.international punkt international +.xin punkt xin +.bike punkt bike +.school punkt school +.stream punkt stream +.frl punkt frl +.tours punkt tours +.audio punkt audio +.legal punkt legal +.chat punkt chat +.xn--wgbl6a punkt xn--wgbl6a +.photo punkt photo +.surf punkt surf +.pink punkt pink +.xn--p1acf punkt xn--p1acf +.kiwi punkt kiwi +.doctor punkt doctor +.museum punkt museum +.science punkt science +.football punkt football +.xn--80asehdb punkt xn--80asehdb +.fail punkt fail +.moda punkt moda +.party punkt party +.rip punkt rip +.fan punkt fan +.loan punkt loan +.works punkt works +.business punkt business +.bayern punkt bayern +.money punkt money +.xn--9dbq2a punkt xn--9dbq2a +.show punkt show +.house punkt house +.futbol punkt futbol +.krd punkt krd +.fyi punkt fyi +.gold punkt gold +.nagoya punkt nagoya +.quebec punkt quebec +.xn--3e0b707e punkt xn--3e0b707e +.reviews punkt reviews +.farm punkt farm +.photography punkt photography +.photos punkt photos +.xn--tckwe punkt xn--tckwe +.cab punkt cab +.army punkt army +.hamburg punkt hamburg +.amsterdam punkt amsterdam +.pictures punkt pictures +.cash punkt cash +.adult punkt adult +.ski punkt ski +.poker punkt poker +.black punkt black +.fashion punkt fashion +.clinic punkt clinic +.gdn punkt gdn +.tatar punkt tatar +.exchange punkt exchange +.date punkt date +.trade punkt trade +.boutique punkt boutique +.shoes punkt shoes +.ren punkt ren +.green punkt green +.band punkt band +.ltda punkt ltda +.istanbul punkt istanbul +.health punkt health +.webcam punkt webcam +.earth punkt earth +.garden punkt garden +.yoga punkt yoga +.wien punkt wien +.ngo punkt ngo +.xn--80adxhks punkt xn--80adxhks +.xn--80aswg punkt xn--80aswg +.xn--fiqs8s punkt xn--fiqs8s +.repair punkt repair +.xn--90a3ac punkt xn--90a3ac +.fitness punkt fitness +.xn--c1avg punkt xn--c1avg +.immo punkt immo +.observer punkt observer +.vin punkt vin +.singles punkt singles +.haus punkt haus +.okinawa punkt okinawa +.corsica punkt corsica +.irish punkt irish +.university punkt university +.casino punkt casino +.how punkt how +.td punkt td +.dance punkt dance +.technology punkt technology +.barcelona punkt barcelona +.nf punkt nf +.golf punkt golf +.wf punkt wf +.estate punkt estate +.yokohama punkt yokohama +.systems punkt systems +.rugby punkt rugby +.wedding punkt wedding +.healthcare punkt healthcare +.koeln punkt koeln +.enterprises punkt enterprises +.film punkt film +.xn--node punkt xn--node +.accountant punkt accountant +.taxi punkt taxi +.tienda punkt tienda +.nr punkt nr +.toys punkt toys +.gent punkt gent +.vision punkt vision +.rent punkt rent +.fans punkt fans +.energy punkt energy +.jetzt punkt jetzt +.cards punkt cards +.domains punkt domains +.marketing punkt marketing +.soy punkt soy +.foundation punkt foundation +.horse punkt horse +.fish punkt fish +.promo punkt promo +.institute punkt institute +.restaurant punkt restaurant +.exposed punkt exposed +.flowers punkt flowers +.gift punkt gift +.jewelry punkt jewelry +.ki punkt ki +.coach punkt coach +.clothing punkt clothing +.fund punkt fund +.finance punkt finance +.dental punkt dental +.forsale punkt forsale +.tirol punkt tirol +.xn--mgbayh7gpa punkt xn--mgbayh7gpa +.family punkt family +.xn--80ao21a punkt xn--80ao21a +.gn punkt gn +.xn--d1acj3b punkt xn--d1acj3b +.madrid punkt madrid +.soccer punkt soccer +.eco punkt eco +.kitchen punkt kitchen +.consulting punkt consulting +.immobilien punkt immobilien +.xn--90ae punkt xn--90ae +.beer punkt beer +.capital punkt capital +.rio punkt rio +.miami punkt miami +.alsace punkt alsace +.archi punkt archi +.ruhr punkt ruhr +.lgbt punkt lgbt +.hosting punkt hosting +.delivery punkt delivery +.limited punkt limited +.properties punkt properties +.baby punkt baby +.bible punkt bible +.schule punkt schule +.bnpparibas punkt bnpparibas +.shopping punkt shopping +.partners punkt partners +.recipes punkt recipes +.bank punkt bank +.lighting punkt lighting +.ong punkt ong +.hm punkt hm +.srl punkt srl +.discount punkt discount +.gifts punkt gifts +.pizza punkt pizza +.vegas punkt vegas +.bot punkt bot +.builders punkt builders +.llc punkt llc +.tax punkt tax +.realty punkt realty +.equipment punkt equipment +.gmbh punkt gmbh +.tattoo punkt tattoo +.homes punkt homes +.saarland punkt saarland +.property punkt property +.kyoto punkt kyoto +.xn--y9a3aq punkt xn--y9a3aq +.solar punkt solar +.weber punkt weber +.gop punkt gop +.mom punkt mom +.fishing punkt fishing +.cricket punkt cricket +.faith punkt faith +.inc punkt inc +.rentals punkt rentals +.build punkt build +.college punkt college +.sharp punkt sharp +.camera punkt camera +.xn--h2brj9c punkt xn--h2brj9c +.cern punkt cern +.sydney punkt sydney +.dhl punkt dhl +.salon punkt salon +.cologne punkt cologne +.cyou punkt cyou +.game punkt game +.vote punkt vote +.flights punkt flights +.glass punkt glass +.melbourne punkt melbourne +.hiphop punkt hiphop +.mba punkt mba +.post punkt post +.voyage punkt voyage +.ventures punkt ventures +.courses punkt courses +.kaufen punkt kaufen +.productions punkt productions +.abudhabi punkt abudhabi +.hospital punkt hospital +.shiksha punkt shiksha +.racing punkt racing +.furniture punkt furniture +.capetown punkt capetown +.aq punkt aq +.engineering punkt engineering +.study punkt study +.luxury punkt luxury +.reisen punkt reisen +.sandvik punkt sandvik +.surgery punkt surgery +.hockey punkt hockey +.audi punkt audi +.contractors punkt contractors +.florist punkt florist +.cheap punkt cheap +.actor punkt actor +.computer punkt computer +.xn--mgbaam7a8h punkt xn--mgbaam7a8h +.coupons punkt coupons +.management punkt management +.organic punkt organic +.cooking punkt cooking +.markets punkt markets +.construction punkt construction +.voto punkt voto +.realestate punkt realestate +.xn--l1acc punkt xn--l1acc +.engineer punkt engineer +.lawyer punkt lawyer +.physio punkt physio +.yandex punkt yandex +.pharmacy punkt pharmacy +.insure punkt insure +.boston punkt boston +.xn--j6w193g punkt xn--j6w193g +.guitars punkt guitars +.saxo punkt saxo +.car punkt car +.osaka punkt osaka +.xn--mk1bu44c punkt xn--mk1bu44c +.industries punkt industries +.tennis punkt tennis +.abbott punkt abbott +.sap punkt sap +.xn--ses554g punkt xn--ses554g +.gw punkt gw +.credit punkt credit +.xn--kpry57d punkt xn--kpry57d +.supplies punkt supplies +.holdings punkt holdings +.ryukyu punkt ryukyu +.diamonds punkt diamonds +.apartments punkt apartments +.associates punkt associates +.cruises punkt cruises +.fk punkt fk +.theater punkt theater +.ceo punkt ceo +.rodeo punkt rodeo +.attorney punkt attorney +.viajes punkt viajes +.google punkt google +.auto punkt auto +.kp punkt kp +.diet punkt diet +.joburg punkt joburg +.gives punkt gives +.villas punkt villas +.financial punkt financial +.sarl punkt sarl +.catering punkt catering +.vacations punkt vacations +.rehab punkt rehab +.xn--fiqz9s punkt xn--fiqz9s +.gu punkt gu +.investments punkt investments +.qpon punkt qpon +.forex punkt forex +.gripe punkt gripe +.stockholm punkt stockholm +.career punkt career +.bingo punkt bingo +.limo punkt limo +.xn--d1alf punkt xn--d1alf +.storage punkt storage +.cleaning punkt cleaning +.dentist punkt dentist +.hiv punkt hiv +.degree punkt degree +.plumbing punkt plumbing +.citic punkt citic +.crs punkt crs +.edeka punkt edeka +.lease punkt lease +.bargains punkt bargains +.durban punkt durban +.condos punkt condos +.loans punkt loans +.xn--wgbh1c punkt xn--wgbh1c +.uol punkt uol +.accountants punkt accountants +.country punkt country +.seat punkt seat +.aws punkt aws +.yachts punkt yachts +.luxe punkt luxe +.barclays punkt barclays +.bradesco punkt bradesco +.xn--55qx5d punkt xn--55qx5d +.maison punkt maison +.kred punkt kred +.xn--czr694b punkt xn--czr694b +.xn--e1a4c punkt xn--e1a4c +.xn--ngbc5azd punkt xn--ngbc5azd +.komatsu punkt komatsu +.reise punkt reise +.xn--kput3i punkt xn--kput3i +.trading punkt trading +.claims punkt claims +.xn--q9jyb4c punkt xn--q9jyb4c +.mortgage punkt mortgage +.arpa punkt arpa +.xn--o3cw4h punkt xn--o3cw4h +.ricoh punkt ricoh +.xn--6qq986b3xl punkt xn--6qq986b3xl +.sener punkt sener +.xn--54b7fta0cc punkt xn--54b7fta0cc +.versicherung punkt versicherung +.mango punkt mango +.charity punkt charity +.feedback punkt feedback +.security punkt security +.ipiranga punkt ipiranga +.xn--qxam punkt xn--qxam +.xn--zfr164b punkt xn--zfr164b +.creditunion punkt creditunion +.afl punkt afl +.christmas punkt christmas +.broker punkt broker +.vodka punkt vodka +.xn--czru2d punkt xn--czru2d +.gay punkt gay +.tickets punkt tickets +.memorial punkt memorial +.xn--t60b56a punkt xn--t60b56a +.neustar punkt neustar +.mma punkt mma +.xn--io0a7i punkt xn--io0a7i +.xn--3ds443g punkt xn--3ds443g +.navy punkt navy +.aquarelle punkt aquarelle +.juegos punkt juegos +.km punkt km +.total punkt total +.democrat punkt democrat +.sucks punkt sucks +.tires punkt tires +.fage punkt fage +.xn--mgbab2bd punkt xn--mgbab2bd +.jll punkt jll +.blackfriday punkt blackfriday +.jcb punkt jcb +.xn--vuq861b punkt xn--vuq861b +.weir punkt weir +.aco punkt aco +.boats punkt boats +.republican punkt republican +.man punkt man +.motorcycles punkt motorcycles +.hsbc punkt hsbc +.abogado punkt abogado +.reit punkt reit +.axa punkt axa +.xn--6frz82g punkt xn--6frz82g +.sky punkt sky +.dvag punkt dvag +.insurance punkt insurance +.kpn punkt kpn +.praxi punkt praxi +.globo punkt globo +.statebank punkt statebank +.xn--fiq228c5hs punkt xn--fiq228c5hs +.bmw punkt bmw +.xn--g2xx48c punkt xn--g2xx48c +.abb punkt abb +.toray punkt toray +.hisamitsu punkt hisamitsu +.philips punkt philips +.schwarz punkt schwarz +.monash punkt monash +.airforce punkt airforce +.erni punkt erni +.latrobe punkt latrobe +.xn--czrs0t punkt xn--czrs0t \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/de/data/electronic/server_name.tsv b/nemo_text_processing/text_normalization/de/data/electronic/server_name.tsv index c05c20585..d2428d841 100644 --- a/nemo_text_processing/text_normalization/de/data/electronic/server_name.tsv +++ b/nemo_text_processing/text_normalization/de/data/electronic/server_name.tsv @@ -9,4 +9,9 @@ orange wanadoo web comcast -aol \ No newline at end of file +aol +signup +brettspielversand +Sinnesbereichen +sinnesbereichen +amazon \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/de/data/whitelist.tsv b/nemo_text_processing/text_normalization/de/data/whitelist.tsv index b956ade12..9ac21da34 100644 --- a/nemo_text_processing/text_normalization/de/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/de/data/whitelist.tsv @@ -5,3 +5,17 @@ Mr. mister Mrs. misses Ms. miss Nr. nummer +2D zwei-D +2-D zwei-D +3D drei-D +3-D drei-D +3-D-Mammogram drei-D-Mammogram +3D-Mammogram drei-D-Mammogram +2-D-Mammogram zwei-D-Mammogram +2D-Mammogram zwei-D-Mammogram +3-D-Mammographie drei-D-Mammographie +3D-Mammographie drei-D-Mammographie +2-D-Mammographie zwei-D-Mammographie +2D-Mammographie zwei-D-Mammographie +3-D-Drucker drei-D-Drucker +3D-Drucker drei-D-Drucker \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/de/taggers/cardinal.py b/nemo_text_processing/text_normalization/de/taggers/cardinal.py index 8f1a13b0c..902a62b3f 100644 --- a/nemo_text_processing/text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/de/taggers/cardinal.py @@ -15,6 +15,8 @@ from collections import defaultdict import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, @@ -23,7 +25,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil AND = "und" @@ -65,7 +66,7 @@ def get_ties_digit(digit_path: str, tie_path: str) -> 'pynini.FstLike': class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. "101" -> cardinal { integer: "ein hundert und zehn" } Args: @@ -165,7 +166,7 @@ def thousand(): self.graph = ( ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) @@ -180,7 +181,7 @@ def thousand(): self.graph_hundred_component_at_least_one_none_zero_digit = ( ((NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT, 0)) - "0" - "1") @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 3 + @ NEMO_DIGIT**3 @ hundred_non_zero() ) | pynini.cross("1", "eins") diff --git a/nemo_text_processing/text_normalization/de/taggers/date.py b/nemo_text_processing/text_normalization/de/taggers/date.py index ab56ad5ce..8c13882d2 100644 --- a/nemo_text_processing/text_normalization/de/taggers/date.py +++ b/nemo_text_processing/text_normalization/de/taggers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, @@ -21,7 +23,6 @@ GraphFst, insert_space, ) -from pynini.lib import pynutil graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))).optimize() graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize() @@ -41,7 +42,7 @@ def get_year_graph(cardinal: GraphFst) -> 'pynini.FstLike': cardinal: cardinal GraphFst """ - year_gt_2000 = (pynini.union("21", "20") + NEMO_DIGIT ** 2) @ cardinal.graph + year_gt_2000 = (pynini.union("21", "20") + NEMO_DIGIT**2) @ cardinal.graph graph_two_digit = delete_leading_zero @ cardinal.two_digit_non_zero hundred = pynutil.insert("hundert") @@ -63,7 +64,7 @@ def get_year_graph(cardinal: GraphFst) -> 'pynini.FstLike': class DateFst(GraphFst): """ - Finite state transducer for classifying date, e.g. + Finite state transducer for classifying date, e.g. "01.04.2010" -> date { day: "erster" month: "april" year: "zwei tausend zehn" preserve_order: true } "1994" -> date { year: "neunzehn vier und neuzig" } "1900" -> date { year: "neunzehn hundert" } diff --git a/nemo_text_processing/text_normalization/de/taggers/decimal.py b/nemo_text_processing/text_normalization/de/taggers/decimal.py index d5cd58ed0..8d1540110 100644 --- a/nemo_text_processing/text_normalization/de/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/de/taggers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space -from pynini.lib import pynutil quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) @@ -26,7 +27,7 @@ def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstL e.g. 1 million -> integer_part: "eine" quantity: "million" e.g. 1.4 million -> integer_part: "eins" fractional_part: "vier" quantity: "million" - Args: + Args: decimal: decimal FST cardinal_up_to_hundred: cardinal FST """ @@ -47,7 +48,7 @@ def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstL class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. + Finite state transducer for classifying decimal, e.g. -11,4006 billion -> decimal { negative: "true" integer_part: "elf" fractional_part: "vier null null sechs" quantity: "billion" preserve_order: true } 1 billion -> decimal { integer_part: "eins" quantity: "billion" preserve_order: true } Args: diff --git a/nemo_text_processing/text_normalization/de/taggers/electronic.py b/nemo_text_processing/text_normalization/de/taggers/electronic.py index 53a859700..22481d8b7 100644 --- a/nemo_text_processing/text_normalization/de/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/de/taggers/electronic.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space -from pynini.lib import pynutil class ElectronicFst(GraphFst): @@ -34,31 +35,48 @@ def __init__(self, deterministic: bool = True): super().__init__(name="electronic", kind="classify", deterministic=deterministic) dot = pynini.accep(".") - accepted_common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] - accepted_common_domains = pynini.union(*accepted_common_domains) - accepted_symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] - accepted_symbols = pynini.union(*accepted_symbols) - dot - accepted_characters = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols) + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + # all symbols + symbols_no_period = pynini.difference(symbols, dot) # alphabet of accepted symbols excluding the '.' + accepted_characters = pynini.closure( + (NEMO_ALPHA | NEMO_DIGIT | symbols_no_period), 1 + ) # alphabet of accepted chars excluding the '.' + all_characters = pynini.closure( + (NEMO_ALPHA | NEMO_DIGIT | symbols), 1 + ) # alphabet of accepted chars including the '.' + + # domains + domain = dot + accepted_characters + domain_graph = ( + pynutil.insert('domain: "') + + (accepted_characters + pynini.closure(domain, 1)) + + dot.ques + + pynutil.insert('"') + ) # email - username = pynutil.insert("username: \"") + accepted_characters + pynutil.insert("\"") + pynini.cross('@', ' ') - domain_graph = accepted_characters + dot + accepted_characters - domain_graph = pynutil.insert("domain: \"") + domain_graph + pynutil.insert("\"") - domain_common_graph = ( - pynutil.insert("domain: \"") - + accepted_characters - + accepted_common_domains - + pynini.closure((accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1) - + pynutil.insert("\"") + username = pynutil.insert('username: "') + all_characters + pynutil.insert('"') + pynini.cross("@", " ") + email = username + domain_graph + + # social media tags + tag = ( + pynini.cross("@", "") + + pynutil.insert('username: "') + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + dot.ques + + pynutil.insert('"') ) - graph = (username + domain_graph) | domain_common_graph # url protocol_start = pynini.accep("https://") | pynini.accep("http://") protocol_end = pynini.accep("www.") protocol = protocol_start | protocol_end | (protocol_start + protocol_end) - protocol = pynutil.insert("protocol: \"") + protocol + pynutil.insert("\"") - graph |= protocol + insert_space + (domain_graph | domain_common_graph) + protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"') + url = protocol + insert_space + (domain_graph) + + graph = url | domain_graph | email | tag self.graph = graph final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) diff --git a/nemo_text_processing/text_normalization/de/taggers/fraction.py b/nemo_text_processing/text_normalization/de/taggers/fraction.py index 71d51c600..31c113842 100644 --- a/nemo_text_processing/text_normalization/de/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/de/taggers/fraction.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/de/taggers/measure.py b/nemo_text_processing/text_normalization/de/taggers/measure.py index c9dbe4120..a46822a0f 100644 --- a/nemo_text_processing/text_normalization/de/taggers/measure.py +++ b/nemo_text_processing/text_normalization/de/taggers/measure.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, @@ -23,8 +26,6 @@ convert_space, insert_space, ) -from pynini.examples import plurals -from pynini.lib import pynutil unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) suppletive = pynini.string_file(get_abs_path("data/measure/suppletive.tsv")) @@ -81,7 +82,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) optional_unit_denominator = pynini.closure( - pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, 0, 1, + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, ) unit_plural = ( diff --git a/nemo_text_processing/text_normalization/de/taggers/money.py b/nemo_text_processing/text_normalization/de/taggers/money.py index bc27d4159..6c32d4706 100644 --- a/nemo_text_processing/text_normalization/de/taggers/money.py +++ b/nemo_text_processing/text_normalization/de/taggers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, @@ -22,7 +24,6 @@ convert_space, insert_space, ) -from pynini.lib import pynutil min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) diff --git a/nemo_text_processing/text_normalization/de/taggers/ordinal.py b/nemo_text_processing/text_normalization/de/taggers/ordinal.py index 564466551..a99e4e4a8 100644 --- a/nemo_text_processing/text_normalization/de/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/de/taggers/ordinal.py @@ -16,13 +16,14 @@ # Russian minimally supervised number grammar. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst + class OrdinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. "2." -> ordinal { integer: "zwei" } } "2tes" -> ordinal { integer: "zwei" } } diff --git a/nemo_text_processing/text_normalization/de/taggers/telephone.py b/nemo_text_processing/text_normalization/de/taggers/telephone.py index f4306b9be..97482a236 100644 --- a/nemo_text_processing/text_normalization/de/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/de/taggers/telephone.py @@ -13,16 +13,17 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space -from pynini.lib import pynutil class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone, which includes country code, number part and extension + Finite state transducer for classifying telephone, which includes country code, number part and extension - E.g + E.g "+49 1234-1233" -> telephone { country_code: "plus neun und vierzig" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true } "(012) 1234-1233" -> telephone { country_code: "null eins zwei" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true } (0**) @@ -44,7 +45,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): numbers_with_single_digits = pynini.closure(graph_digit + insert_space) + graph_digit - two_digit_and_zero = (NEMO_DIGIT ** 2 @ cardinal.two_digit_non_zero) | graph_zero + two_digit_and_zero = (NEMO_DIGIT**2 @ cardinal.two_digit_non_zero) | graph_zero # def add_space_after_two_digit(): # return pynini.closure(two_digit_and_zero + insert_space) + ( # two_digit_and_zero diff --git a/nemo_text_processing/text_normalization/de/taggers/time.py b/nemo_text_processing/text_normalization/de/taggers/time.py index b9e244635..2fe74f5ba 100644 --- a/nemo_text_processing/text_normalization/de/taggers/time.py +++ b/nemo_text_processing/text_normalization/de/taggers/time.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, convert_space, insert_space -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -26,7 +27,7 @@ class TimeFst(GraphFst): "2 Uhr" -> time { hours: "2" } "09:00 Uhr" -> time { hours: "2" } "02:15:10 Uhr" -> time { hours: "2" minutes: "15" seconds: "10"} - + Args: deterministic: if True will provide a single transduction option, for False multiple transduction are generated (used for audio-based normalization) @@ -42,37 +43,37 @@ def __init__(self, deterministic: bool = True): labels_minute_single = [str(x) for x in range(1, 10)] labels_minute_double = [str(x) for x in range(10, 60)] - delete_leading_zero_to_double_digit = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT + delete_leading_zero_to_double_digit = (pynutil.delete("0").ques | (NEMO_DIGIT - "0")) + NEMO_DIGIT graph_hour = pynini.union(*labels_hour) graph_minute_single = pynini.union(*labels_minute_single) graph_minute_double = pynini.union(*labels_minute_double) - final_graph_hour_only = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_hour_only = pynutil.insert('hours: "') + graph_hour + pynutil.insert('"') final_graph_hour = ( - pynutil.insert("hours: \"") + delete_leading_zero_to_double_digit @ graph_hour + pynutil.insert("\"") + pynutil.insert('hours: "') + delete_leading_zero_to_double_digit @ graph_hour + pynutil.insert('"') ) final_graph_minute = ( - pynutil.insert("minutes: \"") + pynutil.insert('minutes: "') + (pynutil.delete("0") + graph_minute_single | graph_minute_double) - + pynutil.insert("\"") + + pynutil.insert('"') ) final_graph_second = ( - pynutil.insert("seconds: \"") + pynutil.insert('seconds: "') + (pynutil.delete("0") + graph_minute_single | graph_minute_double) - + pynutil.insert("\"") + + pynutil.insert('"') ) final_time_zone_optional = pynini.closure( - pynini.accep(" ") + pynutil.insert("zone: \"") + convert_space(time_zone_graph) + pynutil.insert("\""), + pynini.accep(" ") + pynutil.insert('zone: "') + convert_space(time_zone_graph) + pynutil.insert('"'), 0, 1, ) - # 02:30 Uhr + # Accepts the following formats: 02:30 Uhr, 02.30 Uhr, 2:30 Uhr, 2.30 Uhr graph_hm = ( final_graph_hour - + pynutil.delete(":") + + (pynutil.delete(":") | pynutil.delete(".")) + (pynutil.delete("00") | (insert_space + final_graph_minute)) + final_suffix + final_time_zone_optional @@ -82,9 +83,9 @@ def __init__(self, deterministic: bool = True): graph_hms = ( final_graph_hour + pynutil.delete(":") - + (pynini.cross("00", " minutes: \"0\"") | (insert_space + final_graph_minute)) + + (pynini.cross("00", ' minutes: "0"') | (insert_space + final_graph_minute)) + pynutil.delete(":") - + (pynini.cross("00", " seconds: \"0\"") | (insert_space + final_graph_second)) + + (pynini.cross("00", ' seconds: "0"') | (insert_space + final_graph_second)) + final_suffix + final_time_zone_optional + pynutil.insert(" preserve_order: true") diff --git a/nemo_text_processing/text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/de/taggers/tokenize_and_classify.py index 605a88df3..646d7a6b7 100644 --- a/nemo_text_processing/text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/de/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.de.taggers.date import DateFst from nemo_text_processing.text_normalization.de.taggers.decimal import DecimalFst @@ -37,7 +38,7 @@ generator_main, ) from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -69,15 +70,16 @@ def __init__( os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" far_file = os.path.join( - cache_dir, f"_{input_case}_de_tn_{deterministic}_deterministic{whitelist_file}.far" + cache_dir, + f"_{input_case}_de_tn_{deterministic}_deterministic{whitelist_file}.far", ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") self.cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = self.cardinal.fst @@ -91,7 +93,10 @@ def __init__( self.fraction = FractionFst(cardinal=self.cardinal, deterministic=deterministic) fraction_graph = self.fraction.fst self.measure = MeasureFst( - cardinal=self.cardinal, decimal=self.decimal, fraction=self.fraction, deterministic=deterministic + cardinal=self.cardinal, + decimal=self.decimal, + fraction=self.fraction, + deterministic=deterministic, ) measure_graph = self.measure.fst self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic) @@ -103,7 +108,11 @@ def __init__( telephone_graph = self.telephone.fst self.electronic = ElectronicFst(deterministic=deterministic) electronic_graph = self.electronic.fst - self.money = MoneyFst(cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic) + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) money_graph = self.money.fst self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) whitelist_graph = self.whitelist.fst @@ -120,7 +129,7 @@ def __init__( | pynutil.add_weight(decimal_graph, 1.1) | pynutil.add_weight(money_graph, 1.1) | pynutil.add_weight(telephone_graph, 1.1) - | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.11) ) classify |= pynutil.add_weight(word_graph, 100) @@ -131,7 +140,7 @@ def __init__( pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) ) - graph = token_plus_punct + pynini.closure(pynutil.add_weight(delete_extra_space, 1.1) + token_plus_punct) + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) graph = delete_space + graph + delete_space self.fst = graph.optimize() @@ -140,4 +149,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/de/taggers/whitelist.py b/nemo_text_processing/text_normalization/de/taggers/whitelist.py index 4e8829bf1..c56fb138f 100644 --- a/nemo_text_processing/text_normalization/de/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/de/taggers/whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/de/taggers/word.py b/nemo_text_processing/text_normalization/de/taggers/word.py index 16743d28b..7e0df2a95 100644 --- a/nemo_text_processing/text_normalization/de/taggers/word.py +++ b/nemo_text_processing/text_normalization/de/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/de/utils.py b/nemo_text_processing/text_normalization/de/utils.py index 62c663516..0b364938b 100644 --- a/nemo_text_processing/text_normalization/de/utils.py +++ b/nemo_text_processing/text_normalization/de/utils.py @@ -13,9 +13,10 @@ # limitations under the License. import csv -import logging import os +from nemo_text_processing.utils.logging import logger + def get_abs_path(rel_path): """ @@ -23,13 +24,13 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path if not os.path.exists(abs_path): - logging.warning(f'{abs_path} does not exist') + logger.warning(f'{abs_path} does not exist') return abs_path @@ -42,6 +43,6 @@ def load_labels(abs_path): Returns dictionary of mappings """ - label_tsv = open(abs_path, encoding="utf-8") - labels = list(csv.reader(label_tsv, delimiter="\t")) + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) return labels diff --git a/nemo_text_processing/text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/de/verbalizers/cardinal.py index 6adf17dd7..6b6bbf356 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/de/verbalizers/date.py b/nemo_text_processing/text_normalization/de/verbalizers/date.py index 2a1a8ed72..20e775d28 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, @@ -20,7 +22,6 @@ GraphFst, delete_preserve_order, ) -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/de/verbalizers/decimal.py b/nemo_text_processing/text_normalization/de/verbalizers/decimal.py index 4e5ac7f94..b544a2d6c 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.taggers.decimal import quantities from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, @@ -20,13 +22,12 @@ delete_preserve_order, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. - decimal { negative: "true" integer_part: "elf" fractional_part: "vier null sechs" quantity: "billionen" } -> minus elf komma vier null sechs billionen + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "elf" fractional_part: "vier null sechs" quantity: "billionen" } -> minus elf komma vier null sechs billionen decimal { integer_part: "eins" quantity: "billion" } -> eins billion """ diff --git a/nemo_text_processing/text_normalization/de/verbalizers/electronic.py b/nemo_text_processing/text_normalization/de/verbalizers/electronic.py index 184ce01b1..9e530cd41 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/electronic.py @@ -13,15 +13,18 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, NEMO_NOT_QUOTE, NEMO_SIGMA, + NEMO_SPACE, GraphFst, delete_preserve_order, insert_space, ) -from pynini.lib import pynutil class ElectronicFst(GraphFst): @@ -54,21 +57,26 @@ def add_space_after_char(): verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) - user_name = pynutil.delete("username: \"") + add_space_after_char() + pynutil.delete("\"") + user_name = pynutil.delete('username: "') + add_space_after_char() + pynutil.delete('"') user_name @= verbalize_characters - convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common + convert_defaults = (pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | server_common) | domain_common domain = convert_defaults + pynini.closure(insert_space + convert_defaults) domain @= verbalize_characters - domain = pynutil.delete("domain: \"") + domain + pynutil.delete("\"") + domain = pynutil.delete('domain: "') + domain + pynutil.delete('"') protocol = ( - pynutil.delete("protocol: \"") + pynutil.delete('protocol: "') + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) - + pynutil.delete("\"") + + pynutil.delete('"') ) - self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | ( - user_name + pynini.accep(" ") + pynutil.insert("at ") + domain + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert("at ") + domain | (pynutil.insert("at ") + user_name) ) + + # normalizes sentence-final periods following URLs delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + preserve_final_period = pynini.cdrewrite(pynini.cross(" punkt", "."), "", "[EOS]", NEMO_SIGMA) + + delete_tokens = delete_tokens @ preserve_final_period self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/de/verbalizers/fraction.py b/nemo_text_processing/text_normalization/de/verbalizers/fraction.py index 2221a2912..961e4429c 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/fraction.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,7 +22,6 @@ delete_preserve_order, insert_space, ) -from pynini.lib import pynutil class FractionFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/de/verbalizers/measure.py b/nemo_text_processing/text_normalization/de/verbalizers/measure.py index b9bcd190e..675659044 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/measure.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_preserve_order, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): @@ -27,7 +28,7 @@ class MeasureFst(GraphFst): Finite state transducer for verbalizing measure, e.g. measure { cardinal { integer: "zwei" units: "unzen" } } -> "zwei unzen" measure { cardinal { integer_part: "zwei" quantity: "millionen" units: "unzen" } } -> "zwei millionen unzen" - + Args: decimal: decimal GraphFst cardinal: cardinal GraphFst diff --git a/nemo_text_processing/text_normalization/de/verbalizers/money.py b/nemo_text_processing/text_normalization/de/verbalizers/money.py index 94fde2482..ff29e8965 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/de/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/de/verbalizers/ordinal.py index 917337c70..d4ea8eb09 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst -from pynini.lib import pynutil class OrdinalFst(GraphFst): @@ -42,7 +43,10 @@ def __init__(self, deterministic: bool = True): self.ordinal_stem = graph_digit | graph_ties | graph_thousands suffix = pynini.cdrewrite( - pynini.closure(self.ordinal_stem, 0, 1) + convert_rest, "", "[EOS]", NEMO_SIGMA, + pynini.closure(self.ordinal_stem, 0, 1) + convert_rest, + "", + "[EOS]", + NEMO_SIGMA, ).optimize() self.graph = pynini.compose(graph, suffix) self.suffix = suffix diff --git a/nemo_text_processing/text_normalization/de/verbalizers/telephone.py b/nemo_text_processing/text_normalization/de/verbalizers/telephone.py index 25fbeae8c..5bae8fe2d 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/telephone.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/telephone.py @@ -13,14 +13,15 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order + class TelephoneFst(GraphFst): """ Finite state transducer for verbalizing telephone, e.g. - telephone { country_code: "plus neun und vierzig" number_part: "null eins eins eins null null null" } + telephone { country_code: "plus neun und vierzig" number_part: "null eins eins eins null null null" } -> "plus neun und vierzig null eins eins eins null null null" Args: diff --git a/nemo_text_processing/text_normalization/de/verbalizers/time.py b/nemo_text_processing/text_normalization/de/verbalizers/time.py index f6ba49332..a83f09f2a 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, @@ -21,7 +23,6 @@ convert_space, delete_preserve_order, ) -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/de/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/de/verbalizers/verbalize_final.py index 6f47fe35f..6cda902f8 100644 --- a/nemo_text_processing/text_normalization/de/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/de/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.de.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, @@ -24,13 +25,13 @@ generator_main, ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): """ Finite state transducer that verbalizes an entire sentence - + Args: deterministic: if True will provide a single transduction option, for False multiple options (used for audio-based normalization) @@ -47,7 +48,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"de_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst word = WordFst(deterministic=deterministic).fst @@ -67,4 +68,3 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/en/clean_eval_data.py b/nemo_text_processing/text_normalization/en/clean_eval_data.py index 8c33c4fa6..9d0aaed6b 100644 --- a/nemo_text_processing/text_normalization/en/clean_eval_data.py +++ b/nemo_text_processing/text_normalization/en/clean_eval_data.py @@ -16,6 +16,7 @@ from typing import List import regex as re + from nemo_text_processing.text_normalization.data_loader_utils import ( EOS_TYPE, Instance, @@ -23,7 +24,6 @@ training_data_to_sentences, ) - """ This file is for evaluation purposes. filter_loaded_data() cleans data (list of instances) for text normalization. Filters and cleaners can be specified for each semiotic class individually. @@ -67,7 +67,7 @@ def process(self, instance: Instance) -> Instance: Args: processes given instance with process function - + Returns: processed instance if instance belongs to expected class type or original instance """ if instance.token_type != self.class_type: diff --git a/nemo_text_processing/text_normalization/en/data/electronic/cc_cues.tsv b/nemo_text_processing/text_normalization/en/data/electronic/cc_cues.tsv new file mode 100644 index 000000000..c42d25f13 --- /dev/null +++ b/nemo_text_processing/text_normalization/en/data/electronic/cc_cues.tsv @@ -0,0 +1,5 @@ +card ending in +credit card number +credit card +debit card number +debit card \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/en/data/electronic/domain.tsv b/nemo_text_processing/text_normalization/en/data/electronic/domain.tsv index 0e7042c5f..01da3f0bd 100644 --- a/nemo_text_processing/text_normalization/en/data/electronic/domain.tsv +++ b/nemo_text_processing/text_normalization/en/data/electronic/domain.tsv @@ -1,6 +1,7 @@ .com dot com .org dot org .gov dot gov +.edu dot edu .uk dot UK .fr dot FR .net dot net diff --git a/nemo_text_processing/text_normalization/en/data/measure/unit.tsv b/nemo_text_processing/text_normalization/en/data/measure/unit.tsv index c033ab842..6a3ad687d 100644 --- a/nemo_text_processing/text_normalization/en/data/measure/unit.tsv +++ b/nemo_text_processing/text_normalization/en/data/measure/unit.tsv @@ -124,4 +124,12 @@ ps PS s S tb TB tb YB -zb ZB \ No newline at end of file +zb ZB +sec second +min minute +hr hour +d day +wk week +mo month +yr year +svc service \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/en/data/money/per_unit.tsv b/nemo_text_processing/text_normalization/en/data/money/per_unit.tsv index 654806689..40804ca56 100644 --- a/nemo_text_processing/text_normalization/en/data/money/per_unit.tsv +++ b/nemo_text_processing/text_normalization/en/data/money/per_unit.tsv @@ -1,2 +1,70 @@ /ea each -/dozen \ No newline at end of file +/dozen per dozen +/sec per second +/min per minute +/hr per hour +/h per hour +/d per day +/wk per week +/mo per month +/yr per year +/svc per service +/L per liter +/l per liter +/gal per gallon +/gallon per gallon +/kg per kilogram +/g per gram +/lb per pound +/pound per pound +/oz per ounce +/ounce per ounce +/unit per unit +/ft per foot +/foot per foot +/' per foot +/in per inch +/inch per inch +/" per inch +/m per meter +/meter per meter +/yard per yard +/yd per yard +/mile per mile +/mi per mile +/square meter per square meter +/sq meter per square meter +/sq m per square meter +/m² per square meter +/square foot per square foot +/sq foot per square foot +/sq ft per square foot +/ft² per square foot +/cubic foot per cubic foot +/cu foot per cubic foot +/cu ft per cubic foot +/ft³ per cubic foot +/cubic meter per cubic meter +/cu meter per cubic meter +/cu m per cubic meter +/m³ per cubic meter +/visit per visit +/page per page +/pg per page +/usr per user +/user per user +/night per night +/passenger per passenger +/word per word +/wd per word +/GB per gigabyte +/MB per megabyte +/TB per terrabyte +/cycle per cycle +/load per load +/person per person +/task per task +/shift per shift +/project per project +/class per class +/session per session \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv b/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv index e81924755..758691676 100644 --- a/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv +++ b/nemo_text_processing/text_normalization/en/data/whitelist/tts.tsv @@ -98,7 +98,8 @@ b. c. BC b.c. BC A. A. a a A.A. AA -A&A A and A +A&A AANDA +A100 AONEHUNDRED AAAs AAA's AaB AAB Aabo AABO @@ -117,7 +118,7 @@ Abp ABP Abr ABR abv ABV Abz ABZ -A&C A and C +A&C AANDC A. C. AC A.C. AC A/C AC @@ -131,12 +132,12 @@ ACMs ACM's AcpA ACPA AcpB ACPB acq ACQ -A&CR A and CR +A&CR AANDCR AcSOC ACSOC acu ACU AC&W AC and W Acy ACY -A&D A and D +A&D AANDD Adab ADAB A. D. AD A.D. AD @@ -157,7 +158,7 @@ ADRs ADR's ADSRs ADSR's ADSs ADS's Adwa ADWA -A&E A and E +A&E AANDE A. E. AE A.E. AE Aed AED @@ -220,7 +221,7 @@ Ahta AHTA aht AHT Ahu AHU Ahva AHVA -A&I A and I +A&I AANDI A. I. AI A.I. AI Aias AIA's @@ -251,7 +252,7 @@ aka. AKA aka AKA Aka. AKA Aka AKA -A&K A and K +A&K AANDK A. K. AK A.K. AK AKAPs AKAP's @@ -260,13 +261,13 @@ AKAs AKA's akh AKH Akpa AKPA Aku AKU -A&L A and L +A&L AANDL A. L. AL A.L. AL Alh ALH ALUs ALU's A'ma AMA -A&M A and M +A&M AANDM a.m. AM a.m AM A. M. AM @@ -287,7 +288,7 @@ Amm AMM AM&O AM and O Ampt AMPT amr AMR -A&M's A and M's +A&M's AANDM's Ams' AM's amsl AMSL Amta AMTA @@ -305,7 +306,7 @@ Angu ANGU Anhe ANHE Anr ANR Anrep ANREP -A&O A and O +A&O AANDO A. O. AO A.O. AO AOCCs AOCC's @@ -326,7 +327,7 @@ AoS AOS Aotus AOTU's aov AOV aovf AOVF -A&P A and P +A&P AANDP A. P. AP A.P. AP Apc APC @@ -347,14 +348,14 @@ appr APPR Appts APPT's appu APPU Appu APPU -A&P's A and P's +A&P's AANDP's Aps AP's APs AP's AP&T AP and T Aqa AQA Aql AQL Aqr AQR -A&R A and R +A&R AANDR A. R. AR A.R. AR ArgR ARGR @@ -389,7 +390,7 @@ Asst ASST Aste ASTE Astt ASTT Aswa ASWA -A&T A and T +A&T AANDT atac ATAC Atac ATAC Atad ATAD @@ -422,12 +423,12 @@ ATOs ATO's atpB ATPB atri ATRI Atri ATRI -A&T's A and T's +A&T's AANDT's A.T.s AT's AT&SF AT and SF -AT&T AT and T +AT&T ATANDT attd ATTD -AT&T's AT and T's +AT&T's ATANDTS atv ATV ATVs ATV's AtxA ATXA @@ -643,7 +644,7 @@ Bzyb BZYB Caat CAAT C. A. CA C.A. CA -C&A C and A +C&A CANDA CA&CC CA and CC Cadw CADW caeca CAECA @@ -893,7 +894,7 @@ czy CZY Daai DAAI D. A. DA D.A. DA -D&AD D and AD +D&AD DANDAD D.A.N.C.E. dance Dav DAV dBa DBA @@ -1511,7 +1512,7 @@ Geu GEU Gev GEV GeV GEV Gfa GFA -GF&A GF and A +GF&A GFANDA G. F. GF G.F. GF G. G. GG @@ -2052,7 +2053,7 @@ JHSVs JHSV's JHud JHUD J. I. JI J.I. JI -J&J J and J +J&J JANDJ J. J. JJ J.J. JJ Jka JKA @@ -2107,6 +2108,7 @@ Jym JYM Jymn JYMN J. Z. JZ J.Z. JZ +K8S KUBERNETES K. A. KA K.A. KA kbi KBI @@ -2323,6 +2325,7 @@ L.Y. LY L&YR L and YR M. A. MA M.A. MA +MAX-Q MAXQ M&A M and A Mbewu MBEWU mbi MBI @@ -2396,7 +2399,7 @@ M&L M and L M. L. ML M.L. ML MLPs MLP's -MM&A MM and A +MM&A MMANDA M&M M and M M. M. MM M.M. MM @@ -2553,6 +2556,7 @@ N.W. NW nyc NYC N. Y. NY N.Y. NY +NX-SOC NXSOC N'Zif NZIF NZiK NZIK N'Zi NZI @@ -2785,9 +2789,9 @@ Pyw PYW Pyx PYX P. Z. PZ Q. A. QA -Q&A Q and A -Q&A's Q and A's -Q&As Q and A's +Q&A QANDA +Q&A's QANDA's +Q&As QANDAS QbA QBA Q.B. QB Q.C. QC @@ -2923,6 +2927,7 @@ rup RUP Rup RUP R. U. RU R.U. RU +RUN:AI RUN AI RutB RUTB Ruu RUU Ruwa RUWA @@ -2995,7 +3000,7 @@ S. F. SF S.F. SF sfs SFS S.F.'s SF's -SG&A SG and A +SG&A SGANDA sgb SGB Sgip SGIP sgml SGML @@ -3093,6 +3098,7 @@ Ssy SSY ST&AJ ST and AJ STDs STD's stfv STFV +STG-TWO STGTWO STGs STG's STIs STI's STi STI @@ -3849,3 +3855,36 @@ Z. Y. ZY Z.Y. ZY Z. Z. ZZ Z.Z. ZZ +2.5G two point five g +7-eleven seven eleven +AAA triple a +C# c sharp +DeepMind deep mind +DeepStream deep stream +DevOps dev ops +DisplayPort display port +HuggingFace hugging face +ServiceNow service now +SuperCloud super cloud +Wi-Fi wifi +7-Eleven seven eleven +Deepmind deep mind +Deepstream deep stream +Devops dev ops +Displayport display port +Huggingface hugging face +Servicenow service now +Supercloud super cloud +wi-fi wifi +deepmind deep mind +deepstream deep stream +devops dev ops +displayport display port +huggingface hugging face +servicenow service now +supercloud super cloud +wi-fi wifi +3-D three d +401k four oh one k +401(k) four oh one k +401 (k) four oh one k \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/en/graph_utils.py b/nemo_text_processing/text_normalization/en/graph_utils.py index 847cc7252..23ebcf6bd 100644 --- a/nemo_text_processing/text_normalization/en/graph_utils.py +++ b/nemo_text_processing/text_normalization/en/graph_utils.py @@ -13,19 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import string from pathlib import Path from typing import Dict import pynini -from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels from pynini import Far from pynini.examples import plurals from pynini.export import export from pynini.lib import byte, pynutil, utf8 +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from nemo_text_processing.utils.logging import logger + NEMO_CHAR = utf8.VALID_UTF8_CHAR NEMO_DIGIT = byte.DIGIT @@ -34,9 +35,9 @@ NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() NEMO_HEX = pynini.union(*string.hexdigits).optimize() -NEMO_NON_BREAKING_SPACE = u"\u00A0" +NEMO_NON_BREAKING_SPACE = "\u00a0" NEMO_SPACE = " " -NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00A0").optimize() +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() @@ -78,20 +79,60 @@ delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") delete_preserve_order = pynini.closure( pynutil.delete(" preserve_order: true") - | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) ) + +# Common string literals; expand as you see fit. +username_string = "username" +double_quotes = '"' +domain_string = "domain" +protocol_string = "protocol" +slash = "/" +double_slash = "//" +triple_slash = "///" +file = "file" +period = "." +at = "@" +colon = ":" +https = "https" +http = "http" +www = "www" + + suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) # _v = pynini.union("a", "e", "i", "o", "u") _c = pynini.union( - "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z" + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", ) _ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") _es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") _s = NEMO_SIGMA + pynutil.insert("s") graph_plural = plurals._priority_union( - suppletive, plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, ).optimize() SINGULAR_TO_PLURAL = graph_plural @@ -106,8 +147,10 @@ def capitalized_input_graph( - graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None -) -> 'pynini.FstLike': + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": """ Allow graph input to be capitalized, e.g. for ITN) @@ -128,7 +171,7 @@ def capitalized_input_graph( return graph -def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): """ Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. @@ -140,7 +183,7 @@ def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): for rule, graph in graphs.items(): exporter[rule] = graph.optimize() exporter.close() - logging.info(f'Created {file_name}') + logger.info(f"Created {file_name}") def get_plurals(fst): @@ -167,7 +210,7 @@ def get_singulars(fst): return PLURAL_TO_SINGULAR @ fst -def convert_space(fst) -> 'pynini.FstLike': +def convert_space(fst) -> "pynini.FstLike": """ Converts space to nonbreaking space. Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" @@ -190,7 +233,10 @@ def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): written_capitalized = written[0].upper() + written[1:] additional_labels.extend( [ - [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized [ written_capitalized, spoken.upper().replace(" AND ", " and "), @@ -201,10 +247,13 @@ def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): spoken_no_space = spoken.replace(" ", "") # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" if len(spoken) == (2 * len(spoken_no_space) - 1): - logging.debug(f"This is weight {weight}") + logger.debug(f"This is weight {weight}") if len(weight) == 0: additional_labels.extend( - [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] ) else: additional_labels.extend( @@ -236,7 +285,7 @@ def __init__(self, name: str, kind: str, deterministic: bool = True): self._fst = None self.deterministic = deterministic - self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") if self.far_exist(): self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() @@ -247,14 +296,14 @@ def far_exist(self) -> bool: return self.far_path.exists() @property - def fst(self) -> 'pynini.FstLike': + def fst(self) -> "pynini.FstLike": return self._fst @fst.setter def fst(self, fst): self._fst = fst - def add_tokens(self, fst) -> 'pynini.FstLike': + def add_tokens(self, fst) -> "pynini.FstLike": """ Wraps class name around to given fst @@ -266,7 +315,7 @@ def add_tokens(self, fst) -> 'pynini.FstLike': """ return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") - def delete_tokens(self, fst) -> 'pynini.FstLike': + def delete_tokens(self, fst) -> "pynini.FstLike": """ Deletes class name wrap around output of given fst @@ -285,4 +334,4 @@ def delete_tokens(self, fst) -> 'pynini.FstLike': + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/en/taggers/abbreviation.py b/nemo_text_processing/text_normalization/en/taggers/abbreviation.py index 640bb487d..f4abc3352 100644 --- a/nemo_text_processing/text_normalization/en/taggers/abbreviation.py +++ b/nemo_text_processing/text_normalization/en/taggers/abbreviation.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_UPPER, GraphFst, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_UPPER, GraphFst, insert_space + class AbbreviationFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/en/taggers/cardinal.py b/nemo_text_processing/text_normalization/en/taggers/cardinal.py index 4249d6cbd..5e2a8535c 100644 --- a/nemo_text_processing/text_normalization/en/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/en/taggers/cardinal.py @@ -14,6 +14,9 @@ import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, @@ -23,13 +26,11 @@ ) from nemo_text_processing.text_normalization.en.taggers.date import get_four_digit_year_graph from nemo_text_processing.text_normalization.en.utils import get_abs_path -from pynini.examples import plurals -from pynini.lib import pynutil class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. -23 -> cardinal { negative: "true" integer: "twenty three" } } Args: @@ -82,7 +83,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): graph = ( pynini.closure(NEMO_DIGIT, 1, 3) - + (pynini.closure(pynutil.delete(",") + NEMO_DIGIT ** 3) | pynini.closure(NEMO_DIGIT ** 3)) + + (pynini.closure(pynutil.delete(",") + NEMO_DIGIT**3) | pynini.closure(NEMO_DIGIT**3)) ) @ graph self.graph = graph @@ -117,7 +118,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): ) final_graph |= pynini.compose(final_graph, one_to_a_replacement_graph.optimize() + NEMO_SIGMA).optimize() # remove commas for 4 digits numbers - four_digit_comma_graph = (NEMO_DIGIT - "0") + pynutil.delete(",") + NEMO_DIGIT ** 3 + four_digit_comma_graph = (NEMO_DIGIT - "0") + pynutil.delete(",") + NEMO_DIGIT**3 final_graph |= pynini.compose(four_digit_comma_graph.optimize(), final_graph).optimize() self.final_graph = final_graph diff --git a/nemo_text_processing/text_normalization/en/taggers/date.py b/nemo_text_processing/text_normalization/en/taggers/date.py index 93cca2395..52225f0ba 100644 --- a/nemo_text_processing/text_normalization/en/taggers/date.py +++ b/nemo_text_processing/text_normalization/en/taggers/date.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -29,8 +32,6 @@ get_abs_path, load_labels, ) -from pynini.examples import plurals -from pynini.lib import pynutil graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/number/teen.tsv"))).optimize() graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))).optimize() @@ -42,7 +43,7 @@ def get_ties_graph(deterministic: bool = True): """ - Returns two digit transducer, e.g. + Returns two digit transducer, e.g. 03 -> o three 12 -> thirteen 20 -> twenty @@ -118,18 +119,18 @@ def _get_year_graph(cardinal_graph, deterministic: bool = True): Transducer for year, only from 1000 - 2999 e.g. 1290 -> twelve nineteen 2000 - 2009 will be verbalized as two thousand. - + Transducer for 3 digit year, e.g. 123-> one twenty three - + Transducer for year with suffix 123 A.D., 4200 B.C """ graph = get_four_digit_year_graph(deterministic) - graph = (pynini.union("1", "2") + (NEMO_DIGIT ** 3) + pynini.closure(pynini.cross(" s", "s") | "s", 0, 1)) @ graph + graph = (pynini.union("1", "2") + (NEMO_DIGIT**3) + pynini.closure(pynini.cross(" s", "s") | "s", 0, 1)) @ graph graph |= _get_two_digit_year_with_s_graph() - three_digit_year = (NEMO_DIGIT @ cardinal_graph) + insert_space + (NEMO_DIGIT ** 2) @ cardinal_graph + three_digit_year = (NEMO_DIGIT @ cardinal_graph) + insert_space + (NEMO_DIGIT**2) @ cardinal_graph year_with_suffix = ( (get_four_digit_year_graph(deterministic=True) | three_digit_year) + delete_space + insert_space + year_suffix ) @@ -158,7 +159,7 @@ def _get_financial_period_graph(): class DateFst(GraphFst): """ - Finite state transducer for classifying date, e.g. + Finite state transducer for classifying date, e.g. jan. 5, 2012 -> date { month: "january" day: "five" year: "twenty twelve" preserve_order: true } jan. 5 -> date { month: "january" day: "five" preserve_order: true } 5 january 2012 -> date { day: "five" month: "january" year: "twenty twelve" preserve_order: true } @@ -269,7 +270,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool, lm: bool = False): ) graph_dmy = day_graph + delete_extra_space + month_graph + optional_graph_year - day_ex_month = (NEMO_DIGIT ** 2 - pynini.project(month_numbers_graph, "input")) @ day_graph + day_ex_month = (NEMO_DIGIT**2 - pynini.project(month_numbers_graph, "input")) @ day_graph for x in ["-", "/", "."]: delete_sep = pynutil.delete(x) graph_dmy |= ( diff --git a/nemo_text_processing/text_normalization/en/taggers/decimal.py b/nemo_text_processing/text_normalization/en/taggers/decimal.py index 2486b5f8c..f68d7da4b 100644 --- a/nemo_text_processing/text_normalization/en/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/en/taggers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, TO_UPPER, GraphFst, get_abs_path from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, TO_UPPER, GraphFst, get_abs_path + delete_space = pynutil.delete(" ") quantities = pynini.string_file(get_abs_path("data/number/thousand.tsv")) quantities_abbr = pynini.string_file(get_abs_path("data/number/quantity_abbr.tsv")) @@ -30,7 +31,7 @@ def get_quantity( e.g. 1 million -> integer_part: "one" quantity: "million" e.g. 1.5 million -> integer_part: "one" fractional_part: "five" quantity: "million" - Args: + Args: decimal: decimal FST cardinal_up_to_hundred: cardinal FST """ @@ -62,7 +63,7 @@ def get_quantity( class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. + Finite state transducer for classifying decimal, e.g. -12.5006 billion -> decimal { negative: "true" integer_part: "12" fractional_part: "five o o six" quantity: "billion" } 1 billion -> decimal { integer_part: "one" quantity: "billion" } diff --git a/nemo_text_processing/text_normalization/en/taggers/electronic.py b/nemo_text_processing/text_normalization/en/taggers/electronic.py index c347bfe1a..25c3c445a 100644 --- a/nemo_text_processing/text_normalization/en/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/en/taggers/electronic.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,9 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import ( +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( # common string literals MIN_NEG_WEIGHT, NEMO_ALPHA, NEMO_DIGIT, @@ -26,7 +28,6 @@ get_abs_path, insert_space, ) -from pynini.lib import pynutil class ElectronicFst(GraphFst): @@ -48,15 +49,30 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): else: numbers = pynutil.insert(" ") + cardinal.long_numbers + pynutil.insert(" ") + cc_cues = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")), + MIN_NEG_WEIGHT, + ) + + cc_cues = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")), + MIN_NEG_WEIGHT, + ) + accepted_symbols = pynini.project(pynini.string_file(get_abs_path("data/electronic/symbol.tsv")), "input") + accepted_common_domains = pynini.project( pynini.string_file(get_abs_path("data/electronic/domain.tsv")), "input" ) - dict_words = pynutil.add_weight(pynini.string_file(get_abs_path("data/electronic/words.tsv")), MIN_NEG_WEIGHT) + dict_words = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/words.tsv")), + MIN_NEG_WEIGHT, + ) dict_words_without_delimiter = dict_words + pynini.closure( - pynutil.add_weight(pynutil.insert(" ") + dict_words, MIN_NEG_WEIGHT), 1 + pynutil.add_weight(pynutil.insert(" ") + dict_words, MIN_NEG_WEIGHT), + 1, ) dict_words_graph = dict_words_without_delimiter | dict_words @@ -69,54 +85,95 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ).optimize() graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() + username = (NEMO_ALPHA | dict_words_graph) + pynini.closure( NEMO_ALPHA | numbers | accepted_symbols | dict_words_graph ) - username = pynutil.insert("username: \"") + username + pynutil.insert("\"") + pynini.cross('@', ' ') + username = pynutil.insert('username: "') + username + pynutil.insert('"') + pynini.cross("@", " ") domain_graph = all_accepted_symbols_start + pynini.closure( all_accepted_symbols_end | pynutil.add_weight(accepted_common_domains, MIN_NEG_WEIGHT) ) protocol_symbols = pynini.closure((graph_symbols | pynini.cross(":", "colon")) + pynutil.insert(" ")) + protocol_start = (pynini.cross("https", "HTTPS ") | pynini.cross("http", "HTTP ")) + ( pynini.accep("://") @ protocol_symbols ) + protocol_file_start = pynini.accep("file") + insert_space + (pynini.accep(":///") @ protocol_symbols) protocol_end = pynutil.add_weight(pynini.cross("www", "WWW ") + pynini.accep(".") @ protocol_symbols, -1000) + protocol = protocol_file_start | protocol_start | protocol_end | (protocol_start + protocol_end) domain_graph_with_class_tags = ( - pynutil.insert("domain: \"") + pynutil.insert('domain: "') + pynini.compose( NEMO_ALPHA + pynini.closure(NEMO_NOT_SPACE) + (NEMO_ALPHA | NEMO_DIGIT | pynini.accep("/")), domain_graph, ).optimize() - + pynutil.insert("\"") + + pynutil.insert('"') ) - protocol = pynutil.insert("protocol: \"") + pynutil.add_weight(protocol, MIN_NEG_WEIGHT) + pynutil.insert("\"") + protocol = pynutil.insert('protocol: "') + pynutil.add_weight(protocol, MIN_NEG_WEIGHT) + pynutil.insert('"') + # email graph = pynini.compose( NEMO_SIGMA + pynini.accep("@") + NEMO_SIGMA + pynini.accep(".") + NEMO_SIGMA, username + domain_graph_with_class_tags, ) - # abc.com, abc.com/123-sm - # when only domain, make sure it starts and end with NEMO_ALPHA - graph |= ( - pynutil.insert("domain: \"") - + pynini.compose( - NEMO_ALPHA + pynini.closure(NEMO_NOT_SPACE) + accepted_common_domains + pynini.closure(NEMO_NOT_SPACE), - domain_graph, - ).optimize() - + pynutil.insert("\"") - ) + full_stop_accep = pynini.accep(".") + dollar_accep = pynini.accep("$") # Include for the correct transduction of the money graph + excluded_symbols = full_stop_accep | dollar_accep | pynini.accep(",") + filtered_symbols = pynini.difference(accepted_symbols, excluded_symbols) + accepted_characters = NEMO_ALPHA | NEMO_DIGIT | filtered_symbols + domain_component = full_stop_accep + pynini.closure(accepted_characters, 2) + graph_domain = pynutil.add_weight( + pynutil.insert('domain: "') + + (pynini.closure(accepted_characters, 1) + pynini.closure(domain_component, 1)) + + pynutil.insert('"'), + 0.1, + ).optimize() + + graph |= pynutil.add_weight(graph_domain, MIN_NEG_WEIGHT) + # www.abc.com/sdafsdf, or https://www.abc.com/asdfad or www.abc.abc/asdfad graph |= protocol + pynutil.insert(" ") + domain_graph_with_class_tags + # recursively handles the "/" in strings like: + # update/upgrade -> update slash upgrade + # update/upgrade/downgrade -> update slash upgrade slash downgrade + + slash_string = ( + pynini.accep(" ").ques + pynini.accep("/") + pynini.accep(" ").ques + pynini.closure(NEMO_ALPHA, 1) + ) + + graph |= ( + pynutil.insert('domain: "') + + (pynini.closure(NEMO_ALPHA, 1) - "and") + + pynini.closure(slash_string, 1) + + pynutil.insert('"') + ).optimize() + + if deterministic: + # credit card cues + numbers = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + cc_cues + pynutil.insert('" domain: "') + numbers + pynutil.insert('"') + ) + graph |= cc_phrases + + if deterministic: + # credit card cues + numbers = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + cc_cues + pynutil.insert('" domain: "') + numbers + pynutil.insert('"') + ) + graph |= cc_phrases + final_graph = self.add_tokens(graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/en/taggers/fraction.py b/nemo_text_processing/text_normalization/en/taggers/fraction.py index ac6877c22..2b33bd7bb 100644 --- a/nemo_text_processing/text_normalization/en/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/en/taggers/fraction.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, get_abs_path from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, get_abs_path + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/en/taggers/measure.py b/nemo_text_processing/text_normalization/en/taggers/measure.py index 3878069b8..e8d92e1da 100644 --- a/nemo_text_processing/text_normalization/en/taggers/measure.py +++ b/nemo_text_processing/text_normalization/en/taggers/measure.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -32,13 +35,11 @@ from nemo_text_processing.text_normalization.en.taggers.whitelist import get_formats from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels from nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as OrdinalVerbalizer -from pynini.examples import plurals -from pynini.lib import pynutil class MeasureFst(GraphFst): """ - Finite state transducer for classifying measure, suppletive aware, e.g. + Finite state transducer for classifying measure, suppletive aware, e.g. -12kg -> measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } 1kg -> measure { cardinal { integer: "one" } units: "kilogram" } .5kg -> measure { decimal { fractional_part: "five" } units: "kilograms" } @@ -51,7 +52,13 @@ class MeasureFst(GraphFst): for False multiple transduction are generated (used for audio-based normalization) """ - def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): super().__init__(name="measure", kind="classify", deterministic=deterministic) cardinal_graph = cardinal.graph_with_and | self.get_range(cardinal.graph_with_and) @@ -60,30 +67,31 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de graph_unit |= pynini.string_file(get_abs_path("data/measure/unit_alternatives.tsv")) graph_unit |= pynini.compose( - pynini.closure(TO_LOWER, 1) + (NEMO_ALPHA | TO_LOWER) + pynini.closure(NEMO_ALPHA | TO_LOWER), graph_unit + pynini.closure(TO_LOWER, 1) + (NEMO_ALPHA | TO_LOWER) + pynini.closure(NEMO_ALPHA | TO_LOWER), + graph_unit, ).optimize() graph_unit_plural = convert_space(graph_unit @ SINGULAR_TO_PLURAL) graph_unit = convert_space(graph_unit) - optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) graph_unit2 = ( pynini.cross("/", "per") + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit ) optional_graph_unit2 = pynini.closure( - delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, 0, 1, + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, + 0, + 1, ) unit_plural = ( - pynutil.insert("units: \"") - + (graph_unit_plural + optional_graph_unit2 | graph_unit2) - + pynutil.insert("\"") + pynutil.insert('units: "') + (graph_unit_plural + optional_graph_unit2 | graph_unit2) + pynutil.insert('"') ) unit_singular = ( - pynutil.insert("units: \"") + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert("\"") + pynutil.insert('units: "') + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert('"') ) subgraph_decimal = ( @@ -101,18 +109,18 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de + decimal.final_graph_wo_negative + delete_space + pynutil.insert(" } ") - + pynutil.insert("units: \"") + + pynutil.insert('units: "') + pynini.union("AM", "FM") - + pynutil.insert("\"") + + pynutil.insert('"') ) subgraph_cardinal = ( pynutil.insert("cardinal { ") + optional_graph_negative - + pynutil.insert("integer: \"") + + pynutil.insert('integer: "') + ((NEMO_SIGMA - "1") @ cardinal_graph) + delete_space - + pynutil.insert("\"") + + pynutil.insert('"') + pynutil.insert(" } ") + unit_plural ) @@ -120,44 +128,44 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de subgraph_cardinal |= ( pynutil.insert("cardinal { ") + optional_graph_negative - + pynutil.insert("integer: \"") + + pynutil.insert('integer: "') + pynini.cross("1", "one") + delete_space - + pynutil.insert("\"") + + pynutil.insert('"') + pynutil.insert(" } ") + unit_singular ) unit_graph = ( - pynutil.insert("cardinal { integer: \"-\" } units: \"") + pynutil.insert('cardinal { integer: "-" } units: "') + ((pynini.cross("/", "per") + delete_zero_or_one_space) | (pynini.accep("per") + pynutil.delete(" "))) + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit - + pynutil.insert("\" preserve_order: true") + + pynutil.insert('" preserve_order: true') ) decimal_dash_alpha = ( pynutil.insert("decimal { ") + decimal.final_graph_wo_negative - + pynini.cross('-', '') - + pynutil.insert(" } units: \"") + + pynini.cross("-", "") + + pynutil.insert(' } units: "') + pynini.closure(NEMO_ALPHA, 1) - + pynutil.insert("\"") + + pynutil.insert('"') ) decimal_times = ( pynutil.insert("decimal { ") + decimal.final_graph_wo_negative - + pynutil.insert(" } units: \"") - + (pynini.cross(pynini.union('x', "X"), 'x') | pynini.cross(pynini.union('x', "X"), ' times')) - + pynutil.insert("\"") + + pynutil.insert(' } units: "') + + (pynini.cross(pynini.union("x", "X"), "x") | pynini.cross(pynini.union("x", "X"), " times")) + + pynutil.insert('"') ) alpha_dash_decimal = ( - pynutil.insert("units: \"") + pynutil.insert('units: "') + pynini.closure(NEMO_ALPHA, 1) - + pynini.accep('-') - + pynutil.insert("\"") + + pynini.accep("-") + + pynutil.insert('"') + pynutil.insert(" decimal { ") + decimal.final_graph_wo_negative + pynutil.insert(" } preserve_order: true") @@ -169,9 +177,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de address = self.get_address_graph(cardinal) address = ( - pynutil.insert("units: \"address\" cardinal { integer: \"") + pynutil.insert('units: "address" cardinal { integer: "') + address - + pynutil.insert("\" } preserve_order: true") + + pynutil.insert('" } preserve_order: true') ) math_operations = pynini.string_file(get_abs_path("data/measure/math_operation.tsv")) @@ -202,9 +210,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) math = ( - pynutil.insert("units: \"math\" cardinal { integer: \"") - + math - + pynutil.insert("\" } preserve_order: true") + pynutil.insert('units: "math" cardinal { integer: "') + math + pynutil.insert('" } preserve_order: true') ) final_graph = ( subgraph_decimal @@ -251,11 +257,12 @@ def get_address_graph(self, cardinal): ordinal_verbalizer = OrdinalVerbalizer().graph ordinal_tagger = OrdinalTagger(cardinal=cardinal).graph ordinal_num = pynini.compose( - pynutil.insert("integer: \"") + ordinal_tagger + pynutil.insert("\""), ordinal_verbalizer + pynutil.insert('integer: "') + ordinal_tagger + pynutil.insert('"'), + ordinal_verbalizer, ) address_num = NEMO_DIGIT ** (1, 2) @ cardinal.graph_hundred_component_at_least_one_none_zero_digit - address_num += insert_space + NEMO_DIGIT ** 2 @ ( + address_num += insert_space + NEMO_DIGIT**2 @ ( pynini.closure(pynini.cross("0", "zero "), 0, 1) + cardinal.graph_hundred_component_at_least_one_none_zero_digit ) @@ -293,11 +300,33 @@ def get_address_graph(self, cardinal): state = pynini.invert(state_graph) state = pynini.closure(pynini.accep(",") + pynini.accep(NEMO_SPACE) + state, 0, 1) - zip_code = pynini.compose(NEMO_DIGIT ** 5, cardinal.single_digits_graph) - zip_code = pynini.closure(pynini.closure(pynini.accep(","), 0, 1) + pynini.accep(NEMO_SPACE) + zip_code, 0, 1,) + zip_code = pynini.compose(NEMO_DIGIT**5, cardinal.single_digits_graph) + zip_code = pynini.closure( + pynini.closure(pynini.accep(","), 0, 1) + pynini.accep(NEMO_SPACE) + zip_code, + 0, + 1, + ) address = address_num + direction + address_words + pynini.closure(city + state + zip_code, 0, 1) + # Handles sentences ending in terminal period abbreviations, when followed by another sentence + # (e.g. Nancy lived on 1428 Elm St. It was a strange place-> Nancy lived on fourteen twenty eight Elm Street. It was a strange place.) + + hyphen = pynini.accep("-") + alpha_chars = NEMO_ALPHA | hyphen + + address_eos = ( + address_num + + direction + + address_words + + pynini.accep(".") + + pynini.closure(NEMO_SPACE, 1, 2) + + NEMO_UPPER + + pynini.closure(alpha_chars) + ) + + address |= pynutil.add_weight(address_eos, -0.001) + address |= address_num + direction + address_words + pynini.closure(pynini.cross(".", ""), 0, 1) return address diff --git a/nemo_text_processing/text_normalization/en/taggers/money.py b/nemo_text_processing/text_normalization/en/taggers/money.py index 43e26bda7..0687b0c1a 100644 --- a/nemo_text_processing/text_normalization/en/taggers/money.py +++ b/nemo_text_processing/text_normalization/en/taggers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -23,16 +25,16 @@ insert_space, ) from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels -from pynini.lib import pynutil min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) +per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv")) class MoneyFst(GraphFst): """ - Finite state transducer for classifying money, suppletive aware, e.g. + Finite state transducer for classifying money, suppletive aware, e.g. $12.05 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true } $12.0500 -> money { integer_part: "twelve" currency_maj: "dollars" fractional_part: "five" currency_min: "cents" preserve_order: true } $1 -> money { currency_maj: "dollar" integer_part: "one" } @@ -58,14 +60,14 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = maj_unit_plural = convert_space(maj_singular @ SINGULAR_TO_PLURAL) maj_unit_singular = convert_space(maj_singular) - graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_unit_singular + pynutil.insert("\"") - graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_unit_plural + pynutil.insert("\"") + graph_maj_singular = pynutil.insert('currency_maj: "') + maj_unit_singular + pynutil.insert('"') + graph_maj_plural = pynutil.insert('currency_maj: "') + maj_unit_plural + pynutil.insert('"') optional_delete_fractional_zeros = pynini.closure( pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), 0, 1 ) - graph_integer_one = pynutil.insert("integer_part: \"") + pynini.cross("1", "one") + pynutil.insert("\"") + graph_integer_one = pynutil.insert('integer_part: "') + pynini.cross("1", "one") + pynutil.insert('"') # only for decimals where third decimal after comma is non-zero or with quantity decimal_delete_last_zeros = ( pynini.closure(NEMO_DIGIT | pynutil.delete(",")) @@ -80,9 +82,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = graph_maj_plural + insert_space + (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final ) - graph_integer = ( - pynutil.insert("integer_part: \"") + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert("\"") - ) + graph_integer = pynutil.insert('integer_part: "') + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert('"') graph_integer_only = graph_maj_singular + insert_space + graph_integer_one graph_integer_only |= graph_maj_plural + insert_space + graph_integer @@ -100,8 +100,8 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = | ((NEMO_DIGIT - "0") + NEMO_DIGIT) ) - graph_min_singular = pynutil.insert(" currency_min: \"") + min_singular + pynutil.insert("\"") - graph_min_plural = pynutil.insert(" currency_min: \"") + min_plural + pynutil.insert("\"") + graph_min_singular = pynutil.insert(' currency_min: "') + min_singular + pynutil.insert('"') + graph_min_plural = pynutil.insert(' currency_min: "') + min_plural + pynutil.insert('"') # format ** dollars ** cent decimal_graph_with_minor = None integer_graph_reordered = None @@ -112,19 +112,20 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = integer_plus_maj |= graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular integer_plus_maj_with_comma = pynini.compose( - NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | pynutil.delete(",")), integer_plus_maj + NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | pynutil.delete(",")), + integer_plus_maj, ) integer_plus_maj = pynini.compose(pynini.closure(NEMO_DIGIT) - "0", integer_plus_maj) integer_plus_maj |= integer_plus_maj_with_comma graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "one") - graph_fractional_one = pynutil.insert("fractional_part: \"") + graph_fractional_one + pynutil.insert("\"") + graph_fractional_one = pynutil.insert('fractional_part: "') + graph_fractional_one + pynutil.insert('"') graph_fractional = ( two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.graph_hundred_component_at_least_one_none_zero_digit ) - graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") + graph_fractional = pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"') fractional_plus_min = graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural fractional_plus_min |= ( @@ -137,9 +138,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = decimal_graph_with_minor_curr |= pynutil.add_weight( integer_plus_maj + pynini.cross(".", " ") - + pynutil.insert("fractional_part: \"") + + pynutil.insert('fractional_part: "') + two_digits_fractional_part @ cardinal.graph_hundred_component_at_least_one_none_zero_digit - + pynutil.insert("\""), + + pynutil.insert('"'), weight=0.0001, ) default_fraction_graph = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final @@ -182,11 +183,17 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = # weight for SH final_graph |= pynutil.add_weight(decimal_graph_with_minor, -0.0001) + # utilizes morphosyntactic features to append "per units" + graph_per_units = pynutil.insert(' morphosyntactic_features: "') + per_units + pynutil.insert('"') + if not deterministic: final_graph |= integer_graph_reordered | decimal_default_reordered # to handle "$2.00" cases final_graph |= pynini.compose( - NEMO_SIGMA + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), integer_graph_reordered + NEMO_SIGMA + pynutil.delete(".") + pynini.closure(pynutil.delete("0"), 1), + integer_graph_reordered, ) + + final_graph += graph_per_units.ques final_graph = self.add_tokens(final_graph.optimize()) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/en/taggers/ordinal.py b/nemo_text_processing/text_normalization/en/taggers/ordinal.py index bb4cc007f..8687b493c 100644 --- a/nemo_text_processing/text_normalization/en/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/en/taggers/ordinal.py @@ -14,15 +14,16 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst + class OrdinalFst(GraphFst): """ Finite state transducer for classifying ordinal, e.g. 13th -> ordinal { integer: "thirteen" } - + Args: cardinal: CardinalFst deterministic: if True will provide a single transduction option, diff --git a/nemo_text_processing/text_normalization/en/taggers/punctuation.py b/nemo_text_processing/text_normalization/en/taggers/punctuation.py index 769b020ce..3d23a3440 100644 --- a/nemo_text_processing/text_normalization/en/taggers/punctuation.py +++ b/nemo_text_processing/text_normalization/en/taggers/punctuation.py @@ -16,11 +16,12 @@ from unicodedata import category import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst -from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels from pynini.examples import plurals from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + class PunctuationFst(GraphFst): """ @@ -42,7 +43,7 @@ def __init__(self, deterministic: bool = True): chr(i) for i in range(sys.maxunicode) if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude - ] + ] + [r"\[", r"\]"] whitelist_symbols = load_labels(get_abs_path("data/whitelist/symbol.tsv")) whitelist_symbols = [x[0] for x in whitelist_symbols] diff --git a/nemo_text_processing/text_normalization/en/taggers/range.py b/nemo_text_processing/text_normalization/en/taggers/range.py index 92c0c325a..c989e99f5 100644 --- a/nemo_text_processing/text_normalization/en/taggers/range.py +++ b/nemo_text_processing/text_normalization/en/taggers/range.py @@ -14,14 +14,15 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, convert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, convert_space + class RangeFst(GraphFst): """ This class is a composite class of two other class instances - + Args: time: composed tagger and verbalizer date: composed tagger and verbalizer @@ -32,7 +33,12 @@ class RangeFst(GraphFst): """ def __init__( - self, time: GraphFst, date: GraphFst, cardinal: GraphFst, deterministic: bool = True, lm: bool = False, + self, + time: GraphFst, + date: GraphFst, + cardinal: GraphFst, + deterministic: bool = True, + lm: bool = False, ): super().__init__(name="range", kind="classify", deterministic=deterministic) @@ -46,14 +52,14 @@ def __init__( cardinal = cardinal.graph_with_and # YEAR - date_year_four_digit = (NEMO_DIGIT ** 4 + pynini.closure(pynini.accep("s"), 0, 1)) @ date - date_year_two_digit = (NEMO_DIGIT ** 2 + pynini.closure(pynini.accep("s"), 0, 1)) @ date + date_year_four_digit = (NEMO_DIGIT**4 + pynini.closure(pynini.accep("s"), 0, 1)) @ date + date_year_two_digit = (NEMO_DIGIT**2 + pynini.closure(pynini.accep("s"), 0, 1)) @ date year_to_year_graph = ( date_year_four_digit + delete_space + pynini.cross("-", " to ") + delete_space - + (date_year_four_digit | date_year_two_digit | (NEMO_DIGIT ** 2 @ cardinal)) + + (date_year_four_digit | date_year_two_digit | (NEMO_DIGIT**2 @ cardinal)) ) mid_year_graph = pynini.accep("mid") + pynini.cross("-", " ") + (date_year_four_digit | date_year_two_digit) diff --git a/nemo_text_processing/text_normalization/en/taggers/roman.py b/nemo_text_processing/text_normalization/en/taggers/roman.py index e12ee4a2b..d0f394273 100644 --- a/nemo_text_processing/text_normalization/en/taggers/roman.py +++ b/nemo_text_processing/text_normalization/en/taggers/roman.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels -from pynini.lib import pynutil class RomanFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/en/taggers/serial.py b/nemo_text_processing/text_normalization/en/taggers/serial.py index 73b6c4e4f..4a5d6bb9d 100644 --- a/nemo_text_processing/text_normalization/en/taggers/serial.py +++ b/nemo_text_processing/text_normalization/en/taggers/serial.py @@ -14,7 +14,12 @@ import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( + MIN_NEG_WEIGHT, + MIN_POS_WEIGHT, NEMO_ALPHA, NEMO_DIGIT, NEMO_NOT_SPACE, @@ -23,20 +28,67 @@ convert_space, ) from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels -from pynini.examples import plurals -from pynini.lib import pynutil + + +def _leading_zero_graph(cardinal: GraphFst) -> "pynini.FstLike": + return pynini.compose(pynini.accep("0") + pynini.closure(NEMO_DIGIT), cardinal.single_digits_graph).optimize() + + +def _build_serial_graph( + num_graph: "pynini.FstLike", + delimiter: "pynini.FstLike", + alphas: "pynini.FstLike", + ordinal: GraphFst, +) -> "pynini.FstLike": + letter_num = alphas + delimiter + num_graph + num_letter = pynini.closure(num_graph + delimiter, 1) + alphas + next_alpha_or_num = pynini.closure(delimiter + (alphas | num_graph)) + next_alpha_or_num |= pynini.closure( + delimiter + + num_graph + + plurals._priority_union(pynini.accep(" "), pynutil.insert(" "), NEMO_SIGMA).optimize() + + alphas + ) + + serial_graph = letter_num + next_alpha_or_num + serial_graph |= num_letter + next_alpha_or_num + serial_graph |= num_graph + delimiter + num_graph + delimiter + num_graph + pynini.closure(delimiter + num_graph) + + symbols = [x[0] for x in load_labels(get_abs_path("data/whitelist/symbol.tsv"))] + symbols = pynini.union(*symbols) + serial_graph |= pynini.compose(NEMO_SIGMA + symbols + NEMO_SIGMA, num_graph + delimiter + num_graph) + + serial_graph = pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.project(ordinal.graph, "input")), serial_graph + ).optimize() + + serial_graph = pynutil.add_weight(serial_graph, MIN_POS_WEIGHT) + serial_graph |= ( + pynini.closure(NEMO_NOT_SPACE, 1) + (pynini.cross("^2", " squared") | pynini.cross("^3", " cubed")).optimize() + ) + + serial_graph = ( + pynini.closure((serial_graph | num_graph | alphas) + delimiter) + + serial_graph + + pynini.closure(delimiter + (serial_graph | num_graph | alphas)) + ) + return serial_graph.optimize() class SerialFst(GraphFst): """ - This class is a composite class of two other class instances - + Finite state transducer for classifying serial numbers without conventional delimiters. + + Digit normalization within letter-digit tokens follows: + 1. 1-2 digits, or single digits followed by zeros -> cardinal + 2. 3 digits not ending in 00, or 4+ digits -> single-digit reading + 3. Digit-only tokens separated by ``/`` -> cardinal per segment (5+ digits stay single-digit) + Args: - time: composed tagger and verbalizer - date: composed tagger and verbalizer - cardinal: tagger + cardinal: cardinal tagger + ordinal: ordinal tagger (used to exclude ordinal readings) deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) + for False multiple transduction are generated (used for audio-based normalization) lm: whether to use for hybrid LM """ @@ -47,31 +99,56 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = Finite state transducer for classifying serial (handles only cases without delimiters, values with delimiters are handled by default). The serial is a combination of digits, letters and dashes, e.g.: - c325b -> tokens { cardinal { integer: "c three two five b" } } + "H800" -> tokens { name: "H eight hundred" } + "a320b" -> tokens { name: "a three two zero b" } + "12/345/67890" -> tokens { name: "twelve/three hundred forty five/six seven eight nine zero" } + """ if deterministic: - num_graph = pynini.compose(NEMO_DIGIT ** (6, ...), cardinal.single_digits_graph).optimize() - num_graph |= pynini.compose(NEMO_DIGIT ** (1, 5), cardinal.graph).optimize() - # to handle numbers starting with zero - num_graph |= pynini.compose( - pynini.accep("0") + pynini.closure(NEMO_DIGIT), cardinal.single_digits_graph + num_graph_pure = ( + pynini.compose(NEMO_DIGIT ** (1, 3), cardinal.graph) + | pynini.compose(NEMO_DIGIT ** (4, ...), cardinal.single_digits_graph) + | _leading_zero_graph(cardinal) + ).optimize() + + num_graph_alnum = ( + pynini.compose(NEMO_DIGIT, cardinal.graph) + | pynini.compose(NEMO_DIGIT**2, cardinal.graph) + | pynutil.add_weight( + pynini.compose(NEMO_DIGIT + pynini.closure("0", 1), cardinal.graph), MIN_NEG_WEIGHT + ) + | pynini.compose( + pynini.difference(NEMO_DIGIT**3, NEMO_DIGIT + NEMO_DIGIT + "00"), cardinal.single_digits_graph + ) + | pynini.compose(NEMO_DIGIT ** (4, ...), cardinal.single_digits_graph) + | _leading_zero_graph(cardinal) ).optimize() + + num_graph_slash = ( + pynini.compose(NEMO_DIGIT ** (1, 4), cardinal.graph) + | pynini.compose(NEMO_DIGIT ** (5, ...), cardinal.single_digits_graph) + | _leading_zero_graph(cardinal) + ).optimize() + else: - num_graph = cardinal.final_graph + num_graph_pure = cardinal.final_graph + num_graph_alnum = cardinal.final_graph + num_graph_slash = cardinal.final_graph # TODO: "#" doesn't work from the file symbols_graph = pynini.string_file(get_abs_path("data/whitelist/symbol.tsv")).optimize() | pynini.cross( "#", "hash" ) - num_graph |= symbols_graph + num_graph_pure |= symbols_graph + num_graph_alnum |= symbols_graph if not self.deterministic and not lm: - num_graph |= cardinal.single_digits_graph - num_graph |= pynini.compose(num_graph, NEMO_SIGMA + pynutil.delete("hundred ") + NEMO_SIGMA) - # also allow double digits to be pronounced as integer in serial number - num_graph |= pynutil.add_weight( - NEMO_DIGIT ** 2 @ cardinal.graph_hundred_component_at_least_one_none_zero_digit, weight=0.0001 + num_graph_pure |= cardinal.single_digits_graph + num_graph_pure |= pynini.compose(num_graph_pure, NEMO_SIGMA + pynutil.delete("hundred ") + NEMO_SIGMA) + num_graph_pure |= pynutil.add_weight( + NEMO_DIGIT**2 @ cardinal.graph_hundred_component_at_least_one_none_zero_digit, weight=MIN_POS_WEIGHT ) + num_graph_alnum = num_graph_pure # add space between letter and digit/symbol symbols = [x[0] for x in load_labels(get_abs_path("data/whitelist/symbol.tsv"))] @@ -89,44 +166,21 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = delimiter |= pynini.cross("-", " dash ") | pynini.cross("/", " slash ") alphas = pynini.closure(NEMO_ALPHA, 1) - letter_num = alphas + delimiter + num_graph - num_letter = pynini.closure(num_graph + delimiter, 1) + alphas - next_alpha_or_num = pynini.closure(delimiter + (alphas | num_graph)) - next_alpha_or_num |= pynini.closure( - delimiter - + num_graph - + plurals._priority_union(pynini.accep(" "), pynutil.insert(" "), NEMO_SIGMA).optimize() - + alphas - ) - serial_graph = letter_num + next_alpha_or_num - serial_graph |= num_letter + next_alpha_or_num - # numbers only with 2+ delimiters - serial_graph |= ( - num_graph + delimiter + num_graph + delimiter + num_graph + pynini.closure(delimiter + num_graph) - ) - # 2+ symbols - serial_graph |= pynini.compose(NEMO_SIGMA + symbols + NEMO_SIGMA, num_graph + delimiter + num_graph) - - # exclude ordinal numbers from serial options - serial_graph = pynini.compose( - pynini.difference(NEMO_SIGMA, pynini.project(ordinal.graph, "input")), serial_graph - ).optimize() + serial_graph = _build_serial_graph(num_graph_pure, delimiter, alphas, ordinal) + serial_graph_alnum = _build_serial_graph(num_graph_alnum, delimiter, alphas, ordinal) - serial_graph = pynutil.add_weight(serial_graph, 0.0001) - serial_graph |= ( - pynini.closure(NEMO_NOT_SPACE, 1) - + (pynini.cross("^2", " squared") | pynini.cross("^3", " cubed")).optimize() - ) - - # at least one serial graph with alpha numeric value and optional additional serial/num/alpha values - serial_graph = ( - pynini.closure((serial_graph | num_graph | alphas) + delimiter) - + serial_graph - + pynini.closure(delimiter + (serial_graph | num_graph | alphas)) + # Rule 3: tokens that contain only digits and slashes (e.g. 31/31/100, 123/261788/2021). + slash_digit_token = ( + pynini.closure(NEMO_DIGIT, 1) + pynini.accep("/") + pynini.closure(NEMO_DIGIT | pynini.accep("/"), 0) ) + slash_serial = pynini.compose( + slash_digit_token, + pynini.closure(num_graph_slash + pynini.accep("/"), 1) + num_graph_slash, + ).optimize() + serial_graph |= pynutil.add_weight(slash_serial, MIN_NEG_WEIGHT) - serial_graph |= pynini.compose(graph_with_space, serial_graph.optimize()).optimize() + serial_graph |= pynini.compose(graph_with_space, serial_graph_alnum.optimize()).optimize() serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize() # this is not to verbolize "/" as "slash" in cases like "import/export" diff --git a/nemo_text_processing/text_normalization/en/taggers/telephone.py b/nemo_text_processing/text_normalization/en/taggers/telephone.py index 1caedffdd..aa9865928 100644 --- a/nemo_text_processing/text_normalization/en/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/en/taggers/telephone.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -24,16 +26,15 @@ plurals, ) from nemo_text_processing.text_normalization.en.utils import get_abs_path -from pynini.lib import pynutil class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone, and IP, and SSN which includes country code, number part and extension - country code optional: +*** + Finite state transducer for classifying telephone, and IP, and SSN which includes country code, number part and extension + country code optional: +*** number part: ***-***-****, or (***) ***-**** extension optional: 1-9999 - E.g + E.g +1 123-123-5678-1 -> telephone { country_code: "one" number_part: "one two three, one two three, five six seven eight" extension: "one" } 1-800-GO-U-HAUL -> telephone { country_code: "one" number_part: "one, eight hundred GO U HAUL" } Args: diff --git a/nemo_text_processing/text_normalization/en/taggers/time.py b/nemo_text_processing/text_normalization/en/taggers/time.py index 4020996cc..b9e4e824f 100644 --- a/nemo_text_processing/text_normalization/en/taggers/time.py +++ b/nemo_text_processing/text_normalization/en/taggers/time.py @@ -14,6 +14,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, GraphFst, @@ -26,7 +28,6 @@ get_abs_path, load_labels, ) -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -40,7 +41,7 @@ class TimeFst(GraphFst): 02:00 -> time { hours: "two" } 2:00 -> time { hours: "two" } 10:00:05 a.m. -> time { hours: "ten" minutes: "zero" seconds: "five" suffix: "a m" } - + Args: cardinal: CardinalFst deterministic: if True will provide a single transduction option, diff --git a/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py index e4f8e335d..7a253cccc 100644 --- a/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import time import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_WHITE_SPACE, GraphFst, @@ -44,7 +45,7 @@ from nemo_text_processing.text_normalization.en.verbalizers.date import DateFst as vDateFst from nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst as vOrdinalFst from nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTimeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -52,7 +53,7 @@ class ClassifyFst(GraphFst): Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. More details to deployment at NeMo/tools/text_processing_deployment. - + Args: input_case: accepting either "lower_cased" or "cased" input. deterministic: if True will provide a single transduction option, @@ -77,77 +78,83 @@ def __init__( os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" far_file = os.path.join( - cache_dir, f"en_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far" + cache_dir, + f"en_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f'ClassifyFst.fst was restored from {far_file}.') + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") start_time = time.time() cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = cardinal.fst - logging.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") + logger.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") start_time = time.time() ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) ordinal_graph = ordinal.fst - logging.debug(f"ordinal: {time.time() - start_time: .2f}s -- {ordinal_graph.num_states()} nodes") + logger.debug(f"ordinal: {time.time() - start_time: .2f}s -- {ordinal_graph.num_states()} nodes") start_time = time.time() decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) decimal_graph = decimal.fst - logging.debug(f"decimal: {time.time() - start_time: .2f}s -- {decimal_graph.num_states()} nodes") + logger.debug(f"decimal: {time.time() - start_time: .2f}s -- {decimal_graph.num_states()} nodes") start_time = time.time() fraction = FractionFst(deterministic=deterministic, cardinal=cardinal) fraction_graph = fraction.fst - logging.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") + logger.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") start_time = time.time() - measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure = MeasureFst( + cardinal=cardinal, + decimal=decimal, + fraction=fraction, + deterministic=deterministic, + ) measure_graph = measure.fst - logging.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") + logger.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") start_time = time.time() date_graph = DateFst(cardinal=cardinal, deterministic=deterministic).fst - logging.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes") + logger.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes") start_time = time.time() time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst - logging.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") + logger.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") start_time = time.time() telephone_graph = TelephoneFst(deterministic=deterministic).fst - logging.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") + logger.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") start_time = time.time() electonic_graph = ElectronicFst(cardinal=cardinal, deterministic=deterministic).fst - logging.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") + logger.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") start_time = time.time() money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst - logging.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") + logger.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") start_time = time.time() whitelist_graph = WhiteListFst( input_case=input_case, deterministic=deterministic, input_file=whitelist ).fst - logging.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") + logger.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") start_time = time.time() punctuation = PunctuationFst(deterministic=deterministic) punct_graph = punctuation.fst - logging.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") + logger.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") start_time = time.time() word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst - logging.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") + logger.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") start_time = time.time() serial_graph = SerialFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic).fst - logging.debug(f"serial: {time.time() - start_time: .2f}s -- {serial_graph.num_states()} nodes") + logger.debug(f"serial: {time.time() - start_time: .2f}s -- {serial_graph.num_states()} nodes") start_time = time.time() v_time_graph = vTimeFst(deterministic=deterministic).fst @@ -156,9 +163,28 @@ def __init__( time_final = pynini.compose(time_graph, v_time_graph) date_final = pynini.compose(date_graph, v_date_graph) range_graph = RangeFst( - time=time_final, date=date_final, cardinal=cardinal, deterministic=deterministic + time=time_final, + date=date_final, + cardinal=cardinal, + deterministic=deterministic, ).fst - logging.debug(f"range: {time.time() - start_time: .2f}s -- {range_graph.num_states()} nodes") + logger.debug(f"range: {time.time() - start_time: .2f}s -- {range_graph.num_states()} nodes") + + # A quick fix to address money ranges: + # $150-$200 -> one hundred and fifty dollars to two hundred dollars + + dash = (pynutil.insert('name: "') + pynini.cross("-", "to") + pynutil.insert('"')).optimize() + + graph_range_money = pynini.closure( + money_graph + + pynutil.insert(" }") + + pynutil.insert(" tokens { ") + + dash + + pynutil.insert(" } ") + + pynutil.insert("tokens { ") + + money_graph, + 1, + ) classify = ( pynutil.add_weight(whitelist_graph, 1.01) @@ -170,10 +196,11 @@ def __init__( | pynutil.add_weight(ordinal_graph, 1.1) | pynutil.add_weight(money_graph, 1.1) | pynutil.add_weight(telephone_graph, 1.1) - | pynutil.add_weight(electonic_graph, 1.1) + | pynutil.add_weight(electonic_graph, 1.11) | pynutil.add_weight(fraction_graph, 1.1) | pynutil.add_weight(range_graph, 1.1) - | pynutil.add_weight(serial_graph, 1.1001) # should be higher than the rest of the classes + | pynutil.add_weight(serial_graph, 1.12) # should be higher than the rest of the classes + | pynutil.add_weight(graph_range_money, 1.1) ) # roman_graph = RomanFst(deterministic=deterministic).fst @@ -186,7 +213,8 @@ def __init__( punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") punct = pynini.closure( pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) - | (pynutil.insert(" ") + punct), + | (pynutil.insert(" ") + punct) + | punct, 1, ) @@ -211,4 +239,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py b/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py index 82f4332d3..5fc8bdbaf 100644 --- a/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py +++ b/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_lm.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -55,8 +57,7 @@ from nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst as vTelephone from nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTime from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst as vWord -from pynini.examples import plurals -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -64,7 +65,7 @@ class ClassifyFst(GraphFst): Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. More details to deployment at NeMo/tools/text_processing_deployment. - + Args: input_case: accepting either "lower_cased" or "cased" input. deterministic: if True will provide a single transduction option, @@ -95,9 +96,9 @@ def __init__( self.fst = pynini.Far(far_file, mode='r')['tokenize_and_classify'] no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() - logging.info(f'ClassifyFst.fst was restored from {far_file}.') + logger.info(f'ClassifyFst.fst was restored from {far_file}.') else: - logging.info(f'Creating ClassifyFst grammars. This might take some time...') + logger.info(f'Creating ClassifyFst grammars. This might take some time...') # TAGGERS cardinal = CardinalFst(deterministic=True, lm=True) cardinal_tagger = cardinal @@ -224,4 +225,3 @@ def get_token_sem_graph(classify_and_verbalize): if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f'ClassifyFst grammars are saved to {far_file}.') diff --git a/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py b/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py index 41f96b013..239984a80 100644 --- a/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py +++ b/nemo_text_processing/text_normalization/en/taggers/tokenize_and_classify_with_audio.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -56,7 +57,7 @@ from nemo_text_processing.text_normalization.en.verbalizers.telephone import TelephoneFst as vTelephone from nemo_text_processing.text_normalization.en.verbalizers.time import TimeFst as vTime from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst as vWord -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -64,7 +65,7 @@ class ClassifyFst(GraphFst): Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. More details to deployment at NeMo/tools/text_processing_deployment. - + Args: input_case: accepting either "lower_cased" or "cased" input. deterministic: if True will provide a single transduction option, @@ -93,9 +94,9 @@ def __init__( ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode='r')['tokenize_and_classify'] - logging.info(f'ClassifyFst.fst was restored from {far_file}.') + logger.info(f'ClassifyFst.fst was restored from {far_file}.') else: - logging.info(f'Creating ClassifyFst grammars. This might take some time...') + logger.info(f'Creating ClassifyFst grammars. This might take some time...') # TAGGERS cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = cardinal.fst @@ -221,7 +222,6 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f'ClassifyFst grammars are saved to {far_file}.') # to remove normalization options that still contain digits and some special symbols # e.g., "P&E" -> {P and E, P&E}, "P & E" will be removed from the list of normalization options diff --git a/nemo_text_processing/text_normalization/en/taggers/whitelist.py b/nemo_text_processing/text_normalization/en/taggers/whitelist.py index a51edec82..8d19a714e 100644 --- a/nemo_text_processing/text_normalization/en/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/en/taggers/whitelist.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( INPUT_CASED, INPUT_LOWER_CASED, @@ -30,7 +32,6 @@ get_abs_path, load_labels, ) -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/en/taggers/word.py b/nemo_text_processing/text_normalization/en/taggers/word.py index fa6a965aa..b988d4c9d 100644 --- a/nemo_text_processing/text_normalization/en/taggers/word.py +++ b/nemo_text_processing/text_normalization/en/taggers/word.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( MIN_NEG_WEIGHT, NEMO_ALPHA, @@ -24,8 +27,6 @@ get_abs_path, ) from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst -from pynini.examples import plurals -from pynini.lib import pynutil class WordFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/en/utils.py b/nemo_text_processing/text_normalization/en/utils.py index 3a88fd819..a2a765a06 100644 --- a/nemo_text_processing/text_normalization/en/utils.py +++ b/nemo_text_processing/text_normalization/en/utils.py @@ -22,7 +22,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path @@ -37,14 +37,14 @@ def load_labels(abs_path): Returns dictionary of mappings """ - label_tsv = open(abs_path, encoding="utf-8") - labels = list(csv.reader(label_tsv, delimiter="\t")) + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) return labels def augment_labels_with_punct_at_end(labels): """ - augments labels: if key ends on a punctuation that value does not have, add a new label + augments labels: if key ends on a punctuation that value does not have, add a new label where the value maintains the punctuation Args: diff --git a/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py b/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py index 191792431..95713b20d 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/abbreviation.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class AbbreviationFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py index eab85015d..a398f892a 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/en/verbalizers/date.py b/nemo_text_processing/text_normalization/en/verbalizers/date.py index 6a2cd12b9..548f18219 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/date.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,8 +23,6 @@ delete_extra_space, delete_space, ) -from pynini.examples import plurals -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/en/verbalizers/decimal.py b/nemo_text_processing/text_normalization/en/verbalizers/decimal.py index 9285edf20..6aa3e9af0 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/en/verbalizers/electronic.py b/nemo_text_processing/text_normalization/en/verbalizers/electronic.py index 1e3fc9b50..98afc3f24 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/electronic.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( MIN_NEG_WEIGHT, NEMO_ALPHA, @@ -28,8 +31,6 @@ insert_space, ) from nemo_text_processing.text_normalization.en.utils import get_abs_path -from pynini.examples import plurals -from pynini.lib import pynutil class ElectronicFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/en/verbalizers/fraction.py b/nemo_text_processing/text_normalization/en/verbalizers/fraction.py index d0c5dc2b1..b4b501210 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/fraction.py @@ -13,11 +13,12 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space -from nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst from pynini.examples import plurals from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space +from nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/en/verbalizers/measure.py b/nemo_text_processing/text_normalization/en/verbalizers/measure.py index 8f9bd2108..c998a809f 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): @@ -29,7 +30,7 @@ class MeasureFst(GraphFst): measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } -> minus twelve kilograms measure { decimal { integer_part: "twelve" fractional_part: "five" } units: "kilograms" } -> twelve point five kilograms tokens { measure { units: "covid" decimal { integer_part: "nineteen" fractional_part: "five" } } } -> covid nineteen point five - + Args: decimal: DecimalFst cardinal: CardinalFst diff --git a/nemo_text_processing/text_normalization/en/verbalizers/money.py b/nemo_text_processing/text_normalization/en/verbalizers/money.py index b3cbc4a31..5b80267ba 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/money.py @@ -13,13 +13,18 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_preserve_order, ) -from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.utils import get_abs_path + +per_units = pynini.string_file(get_abs_path("data/money/per_unit.tsv")) class MoneyFst(GraphFst): @@ -36,11 +41,11 @@ class MoneyFst(GraphFst): def __init__(self, decimal: GraphFst, deterministic: bool = True): super().__init__(name="money", kind="verbalize", deterministic=deterministic) keep_space = pynini.accep(" ") - maj = pynutil.delete("currency_maj: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") - min = pynutil.delete("currency_min: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + maj = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + min = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') fractional_part = ( - pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ) integer_part = decimal.integer @@ -67,5 +72,14 @@ def __init__(self, decimal: GraphFst, deterministic: bool = True): if not deterministic: graph |= graph_integer + delete_preserve_order + per_units_normalized = pynini.project(per_units, "output") + remove_per_units_normalized = ( + pynutil.delete(' morphosyntactic_features: "') + + pynutil.insert(" ") + + per_units_normalized + + pynutil.delete('" ') + ) + graph += remove_per_units_normalized.ques + delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py index c64579ae5..dff205f8e 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/ordinal.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space from nemo_text_processing.text_normalization.en.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): @@ -45,7 +46,10 @@ def __init__(self, deterministic: bool = True): convert_rest = pynutil.insert("th") suffix = pynini.cdrewrite( - graph_digit | graph_teens | pynini.cross("ty", "tieth") | convert_rest, "", "[EOS]", NEMO_SIGMA, + graph_digit | graph_teens | pynini.cross("ty", "tieth") | convert_rest, + "", + "[EOS]", + NEMO_SIGMA, ).optimize() self.graph = pynini.compose(graph, suffix) self.suffix = suffix diff --git a/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py index 6cb7eb0cc..33a472656 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/post_processing.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( MIN_NEG_WEIGHT, NEMO_ALPHA, @@ -25,7 +26,7 @@ generator_main, ) from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class PostProcessingFst: @@ -46,7 +47,7 @@ def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): far_file = os.path.join(cache_dir, "en_tn_post_processing.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] - logging.info(f'Post processing graph was restored from {far_file}.') + logger.info(f'Post processing graph was restored from {far_file}.') else: self.set_punct_dict() self.fst = self.get_punct_postprocess_graph() @@ -98,17 +99,17 @@ def set_punct_dict(self): def get_punct_postprocess_graph(self): """ - Returns graph to post process punctuation marks. + Returns graph to post process punctuation marks. - {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. - By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. """ punct_marks_all = PunctuationFst().punct_marks # no_space_before_punct assume no space before them quotes = ["'", "\"", "``", "«"] dashes = ["-", "—"] - brackets = ["<", "{", "("] + brackets = ["<", "{", "(", r"\["] open_close_single_quotes = [ ("`", "`"), ] diff --git a/nemo_text_processing/text_normalization/en/verbalizers/roman.py b/nemo_text_processing/text_normalization/en/verbalizers/roman.py index 43faebe76..72ab7eee8 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/roman.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/roman.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from nemo_text_processing.text_normalization.en.verbalizers.ordinal import OrdinalFst -from pynini.lib import pynutil class RomanFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/en/verbalizers/telephone.py b/nemo_text_processing/text_normalization/en/verbalizers/telephone.py index 4af7bbb87..0fb51bcba 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/telephone.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/telephone.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + class TelephoneFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/en/verbalizers/time.py b/nemo_text_processing/text_normalization/en/verbalizers/time.py index 518c7dfa2..ff37fd213 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -85,12 +86,12 @@ def __init__(self, deterministic: bool = True): + optional_suffix + optional_zone ) + graph_hms @= pynini.cdrewrite(pynini.cross("one hours", "one hour"), "[BOS]", "", NEMO_SIGMA) graph_hms @= pynini.cdrewrite( pynutil.delete("o ") - | pynini.cross("one minutes", "one minute") - | pynini.cross("one seconds", "one second") - | pynini.cross("one hours", "one hour"), - pynini.union(" ", "[BOS]"), + | pynutil.add_weight(pynini.cross("o one minutes", "one minute"), -0.01) + | pynutil.add_weight(pynini.cross("o one seconds", "one second"), -0.01), + " ", "", NEMO_SIGMA, ) diff --git a/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py index 8dbefc864..6c4fdbc8f 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -24,7 +25,7 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): @@ -48,7 +49,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"en_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst word = WordFst(deterministic=deterministic).fst @@ -72,4 +73,3 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py index 96aa2075b..559605e63 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/whitelist.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WhiteListFst(GraphFst): """ @@ -35,5 +36,5 @@ def __init__(self, deterministic: bool = True): + pynini.closure(NEMO_CHAR - " ", 1) + pynutil.delete("\"") ) - graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/en/verbalizers/word.py b/nemo_text_processing/text_normalization/en/verbalizers/word.py index e124f42ff..b5978030e 100644 --- a/nemo_text_processing/text_normalization/en/verbalizers/word.py +++ b/nemo_text_processing/text_normalization/en/verbalizers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + class WordFst(GraphFst): """ @@ -30,6 +31,6 @@ def __init__(self, deterministic: bool = True): super().__init__(name="word", kind="verbalize", deterministic=deterministic) chars = pynini.closure(NEMO_CHAR - " ", 1) char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") - graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv b/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv new file mode 100644 index 000000000..688da504d --- /dev/null +++ b/nemo_text_processing/text_normalization/es/data/ordinals/roman_exceptions.tsv @@ -0,0 +1,10 @@ +di +mi +vi +i +v +x +l +c +d +m \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/es/data/whitelist.tsv b/nemo_text_processing/text_normalization/es/data/whitelist.tsv index 4be0beb62..f86e6d1a5 100644 --- a/nemo_text_processing/text_normalization/es/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/es/data/whitelist.tsv @@ -14,7 +14,6 @@ dr. doctor dra. doctora d. don da. doña -E este EE. UU. Estados Unidos ee. uu. estados unidos Gob. gobierno @@ -25,7 +24,6 @@ Avda. avenida av. avenida avda. avenida avd. avenida -O oeste pág. página p.ej. por ejemplo p. ej. por ejemplo diff --git a/nemo_text_processing/text_normalization/es/graph_utils.py b/nemo_text_processing/text_normalization/es/graph_utils.py index f9f3dc22b..946f4234e 100644 --- a/nemo_text_processing/text_normalization/es/graph_utils.py +++ b/nemo_text_processing/text_normalization/es/graph_utils.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, NEMO_SPACE from nemo_text_processing.text_normalization.es import LOCALIZATION from nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels -from pynini.lib import pynutil digits = pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "input") tens = pynini.project(pynini.string_file(get_abs_path("data/numbers/ties.tsv")), "input") @@ -39,7 +40,11 @@ fem_hundreds = hundreds @ pynini.cdrewrite(pynini.cross("ientos", "ientas"), "", "", NEMO_SIGMA) -def strip_accent(fst: 'pynini.FstLike') -> 'pynini.FstLike': +ES_MINUS = pynini.union("menos", "Menos", "MENOS").optimize() +ES_PLUS = pynini.union("más", "Más", "MÁS").optimize() + + +def strip_accent(fst: "pynini.FstLike") -> "pynini.FstLike": """ Converts all accented vowels to non-accented equivalents @@ -49,7 +54,7 @@ def strip_accent(fst: 'pynini.FstLike') -> 'pynini.FstLike': return fst @ pynini.cdrewrite(accents, "", "", NEMO_SIGMA) -def shift_cardinal_gender(fst: 'pynini.FstLike') -> 'pynini.FstLike': +def shift_cardinal_gender(fst: "pynini.FstLike") -> "pynini.FstLike": """ Applies gender conversion rules to a cardinal string. These include: rendering all masculine forms of "uno" (including apocopated forms) as "una" and Converting all gendered numbers in the hundreds series (200,300,400...) to feminine equivalent (e.g. "doscientos" -> "doscientas"). Conversion only applies @@ -71,23 +76,23 @@ def shift_cardinal_gender(fst: 'pynini.FstLike') -> 'pynini.FstLike': + (pynini.accep("mil") | pynini.accep("milésimo")) + pynini.closure(NEMO_SPACE + hundreds, 0, 1) + pynini.closure(NEMO_SPACE + one_to_one_hundred, 0, 1) - + pynini.union(pynini.accep("[EOS]"), pynini.accep("\""), decimal_separator) + + pynini.union(pynini.accep("[EOS]"), pynini.accep('"'), decimal_separator) ) before_double_digits = pynini.closure(NEMO_SPACE + one_to_one_hundred, 0, 1) + pynini.union( - pynini.accep("[EOS]"), pynini.accep("\"") + pynini.accep("[EOS]"), pynini.accep('"') ) fem_allign = pynini.cdrewrite(fem_hundreds, "", before_mil, NEMO_SIGMA) # doscientas mil dosciento fem_allign @= pynini.cdrewrite(fem_hundreds, "", before_double_digits, NEMO_SIGMA) # doscientas mil doscienta fem_allign @= pynini.cdrewrite( - fem_ones, "", pynini.union("[EOS]", "\"", decimal_separator), NEMO_SIGMA + fem_ones, "", pynini.union("[EOS]", '"', decimal_separator), NEMO_SIGMA ) # If before a quote or EOS, we know it's the end of a string return fst @ fem_allign -def shift_number_gender(fst: 'pynini.FstLike') -> 'pynini.FstLike': +def shift_number_gender(fst: "pynini.FstLike") -> "pynini.FstLike": """ Performs gender conversion on all verbalized numbers in output. All values in the hundreds series (200,300,400) are changed to feminine gender (e.g. "doscientos" -> "doscientas") and all forms of "uno" (including apocopated forms) are converted to "una". @@ -102,13 +107,16 @@ def shift_number_gender(fst: 'pynini.FstLike') -> 'pynini.FstLike': """ fem_allign = pynini.cdrewrite(fem_hundreds, "", "", NEMO_SIGMA) fem_allign @= pynini.cdrewrite( - fem_ones, "", pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep("\"")), NEMO_SIGMA + fem_ones, + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, ) # If before a quote or EOS, we know it's the end of a string return fst @ fem_allign -def strip_cardinal_apocope(fst: 'pynini.FstLike') -> 'pynini.FstLike': +def strip_cardinal_apocope(fst: "pynini.FstLike") -> "pynini.FstLike": """ Reverts apocope on cardinal strings in line with formation rules. e.g. "un" -> "uno". Due to cardinal formation rules, this in effect only affects strings where the final value is a variation of "un". @@ -121,11 +129,11 @@ def strip_cardinal_apocope(fst: 'pynini.FstLike') -> 'pynini.FstLike': """ # Since cardinals use apocope by default for large values (e.g. "millón"), this only needs to act on the last instance of one strip = pynini.cross("un", "uno") | pynini.cross("ún", "uno") - strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", "\""), NEMO_SIGMA) + strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), NEMO_SIGMA) return fst @ strip -def add_cardinal_apocope_fem(fst: 'pynini.FstLike') -> 'pynini.FstLike': +def add_cardinal_apocope_fem(fst: "pynini.FstLike") -> "pynini.FstLike": """ Adds apocope on cardinal strings in line with stressing rules. e.g. "una" -> "un". This only occurs when "una" precedes a stressed "a" sound in formal speech. This is not predictable with text string, so is included for non-deterministic cases. @@ -138,11 +146,11 @@ def add_cardinal_apocope_fem(fst: 'pynini.FstLike') -> 'pynini.FstLike': """ # Since the stress trigger follows the cardinal string and only affects the preceding sound, this only needs to act on the last instance of one strip = pynini.cross("una", "un") | pynini.cross("veintiuna", "veintiún") - strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", "\""), NEMO_SIGMA) + strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), NEMO_SIGMA) return fst @ strip -def roman_to_int(fst: 'pynini.FstLike') -> 'pynini.FstLike': +def roman_to_int(fst: "pynini.FstLike") -> "pynini.FstLike": """ Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000. e.g. @@ -153,29 +161,57 @@ def roman_to_int(fst: 'pynini.FstLike') -> 'pynini.FstLike': fst: Any fst. Composes fst onto Roman conversion outputs. """ - def _load_roman(file: str): + def _load_roman(file: str, upper_casing: bool): roman = load_labels(get_abs_path(file)) - roman_numerals = [(x, y) for x, y in roman] + [(x.upper(), y) for x, y in roman] + if upper_casing: + roman_numerals = [(x.upper(), y) for x, y in roman] + else: + roman_numerals = [(x, y) for x, y in roman] return pynini.string_map(roman_numerals) - digit = _load_roman("data/roman/digit.tsv") - ties = _load_roman("data/roman/ties.tsv") - hundreds = _load_roman("data/roman/hundreds.tsv") - thousands = _load_roman("data/roman/thousands.tsv") + # A split between all upper-case and all lower-case Roman numerals is introduced in order to preserve orthographic accuracy, + # and to prevent cases in which certain proper nouns e.g. (Li, Xi, Yi, etc.) are transduced to Roman numerals. + + digit_lower = _load_roman("data/roman/digit.tsv", False) + digit_upper = _load_roman("data/roman/digit.tsv", True) + ties_lower = _load_roman("data/roman/ties.tsv", False) + ties_upper = _load_roman("data/roman/ties.tsv", True) + hundreds_lower = _load_roman("data/roman/hundreds.tsv", False) + hundreds_upper = _load_roman("data/roman/hundreds.tsv", True) + thousands_lower = _load_roman("data/roman/thousands.tsv", False) + thousands_upper = _load_roman("data/roman/thousands.tsv", True) graph = ( - digit - | ties + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) + (digit_upper | digit_lower) + | ( + (ties_upper + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01))) + | (ties_lower + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01))) + ) | ( - hundreds - + (ties | pynutil.add_weight(pynutil.insert("0"), 0.01)) - + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ( + hundreds_upper + + (ties_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + | ( + hundreds_lower + + (ties_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) ) | ( - thousands - + (hundreds | pynutil.add_weight(pynutil.insert("0"), 0.01)) - + (ties | pynutil.add_weight(pynutil.insert("0"), 0.01)) - + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ( + thousands_upper + + (hundreds_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (ties_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_upper | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) + | ( + thousands_lower + + (hundreds_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (ties_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + + (digit_lower | pynutil.add_weight(pynutil.insert("0"), 0.01)) + ) ) ).optimize() diff --git a/nemo_text_processing/text_normalization/es/taggers/cardinal.py b/nemo_text_processing/text_normalization/es/taggers/cardinal.py index c535100f7..85402089f 100644 --- a/nemo_text_processing/text_normalization/es/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/es/taggers/cardinal.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import cardinal_separator from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) @@ -46,7 +47,7 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': Returns: fst: A pynini.FstLike object """ - exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string cardinal_string = pynini.closure( @@ -156,7 +157,7 @@ def __init__(self, deterministic: bool = True): self.graph = ( ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/es/taggers/date.py b/nemo_text_processing/text_normalization/es/taggers/date.py index 63c76ff56..dd5cd7f0e 100644 --- a/nemo_text_processing/text_normalization/es/taggers/date.py +++ b/nemo_text_processing/text_normalization/es/taggers/date.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_extra_space from nemo_text_processing.text_normalization.es.graph_utils import roman_to_int, strip_cardinal_apocope from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil articles = pynini.union("de", "del", "el", "del año", "año") delete_leading_zero = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT @@ -115,7 +116,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool): dash = "-" day_optional = pynini.closure(pynini.cross(dash, NEMO_SPACE) + day, 0, 1) - graph_ymd = NEMO_DIGIT ** 4 @ year_only + pynini.cross(dash, NEMO_SPACE) + month_number + day_optional + graph_ymd = NEMO_DIGIT**4 @ year_only + pynini.cross(dash, NEMO_SPACE) + month_number + day_optional final_graph = graph_dmy + pynutil.insert(" preserve_order: true") final_graph |= graph_ymd diff --git a/nemo_text_processing/text_normalization/es/taggers/decimals.py b/nemo_text_processing/text_normalization/es/taggers/decimals.py index 730643d4b..2a3b9c560 100644 --- a/nemo_text_processing/text_normalization/es/taggers/decimals.py +++ b/nemo_text_processing/text_normalization/es/taggers/decimals.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -26,7 +28,6 @@ strip_cardinal_apocope, ) from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) diff --git a/nemo_text_processing/text_normalization/es/taggers/electronic.py b/nemo_text_processing/text_normalization/es/taggers/electronic.py index 13833bbe0..42f01ba4d 100644 --- a/nemo_text_processing/text_normalization/es/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/es/taggers/electronic.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space -from nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + at, + colon, + domain_string, + double_quotes, + double_slash, + http, + https, + protocol_string, + username_string, + www, +) +from nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels + common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] -symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] class ElectronicFst(GraphFst): @@ -34,40 +50,57 @@ class ElectronicFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="electronic", kind="classify", deterministic=deterministic) - dot = pynini.accep(".") - accepted_common_domains = pynini.union(*common_domains) - accepted_symbols = pynini.union(*symbols) - dot - accepted_characters = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols) - acceepted_characters_with_dot = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols | dot) + full_stop_accep = pynini.accep(".") + full_stop = "." + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_full_stop = pynini.difference(symbols, full_stop_accep) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_full_stop), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) + + # domains + domain = full_stop_accep + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters + pynini.closure(domain, 1)) + + pynutil.insert(double_quotes) + ) # email username = ( - pynutil.insert("username: \"") - + acceepted_characters_with_dot - + pynutil.insert("\"") - + pynini.cross('@', ' ') + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + all_characters + + pynutil.insert(double_quotes) + + pynini.cross(at, NEMO_SPACE) ) - domain_graph = accepted_characters + dot + accepted_characters - domain_graph = pynutil.insert("domain: \"") + domain_graph + pynutil.insert("\"") - domain_common_graph = ( - pynutil.insert("domain: \"") - + accepted_characters - + accepted_common_domains - + pynini.closure((accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1) - + pynutil.insert("\"") + email = username + domain_graph + + # social media tags + tag = ( + pynini.cross(at, "") + + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + pynutil.insert(double_quotes) ) - graph = (username + domain_graph) | domain_common_graph # url - protocol_start = pynini.accep("https://") | pynini.accep("http://") + protocol_start = pynini.accep(https + colon + double_slash) | pynini.accep(http + colon + double_slash) + # protocol_end = pynini.accep("www.") protocol_end = ( - pynini.accep("www.") + pynini.accep(www + full_stop) if deterministic - else pynini.accep("www.") | pynini.cross("www.", "doble ve doble ve doble ve.") + else pynini.accep(www + full_stop) | pynini.cross(www + full_stop, "doble ve doble ve doble ve.") ) protocol = protocol_start | protocol_end | (protocol_start + protocol_end) - protocol = pynutil.insert("protocol: \"") + protocol + pynutil.insert("\"") - graph |= protocol + insert_space + (domain_graph | domain_common_graph) + protocol = ( + pynutil.insert(protocol_string + colon + NEMO_SPACE + double_quotes) + + protocol + + pynutil.insert(double_quotes) + ) + url = protocol + pynutil.insert(NEMO_SPACE) + (domain_graph) + + graph = url | domain_graph | email | tag self.graph = graph final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) diff --git a/nemo_text_processing/text_normalization/es/taggers/fraction.py b/nemo_text_processing/text_normalization/es/taggers/fraction.py index 436b4fa20..7bbe86402 100644 --- a/nemo_text_processing/text_normalization/es/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/es/taggers/fraction.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -20,7 +22,6 @@ GraphFst, ) from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil ordinal_exceptions = pynini.string_file(get_abs_path("data/fractions/ordinal_exceptions.tsv")) higher_powers_of_ten = pynini.string_file(get_abs_path("data/fractions/powers_of_ten.tsv")) @@ -46,15 +47,50 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = ordinal_graph = ordinal.graph # 2-10 are all ordinals - three_to_ten = pynini.string_map(["2", "3", "4", "5", "6", "7", "8", "9", "10",]) + three_to_ten = pynini.string_map( + [ + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + ] + ) block_three_to_ten = pynutil.delete(three_to_ten) # To block cardinal productions if not deterministic: # Multiples of tens are sometimes rendered as ordinals - three_to_ten |= pynini.string_map(["20", "30", "40", "50", "60", "70", "80", "90",]) + three_to_ten |= pynini.string_map( + [ + "20", + "30", + "40", + "50", + "60", + "70", + "80", + "90", + ] + ) graph_three_to_ten = three_to_ten @ ordinal_graph graph_three_to_ten @= pynini.cdrewrite(ordinal_exceptions, "", "", NEMO_SIGMA) # Higher powers of tens (and multiples) are converted to ordinals. - hundreds = pynini.string_map(["100", "200", "300", "400", "500", "600", "700", "800", "900",]) + hundreds = pynini.string_map( + [ + "100", + "200", + "300", + "400", + "500", + "600", + "700", + "800", + "900", + ] + ) graph_hundreds = hundreds @ ordinal_graph multiples_of_thousand = ordinal.multiples_of_thousand # So we can have X milésimos @@ -67,7 +103,10 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = graph_higher_powers_of_ten += higher_powers_of_ten graph_higher_powers_of_ten = cardinal_graph @ graph_higher_powers_of_ten graph_higher_powers_of_ten @= pynini.cdrewrite( - pynutil.delete("un "), pynini.accep("[BOS]"), pynini.project(higher_powers_of_ten, "output"), NEMO_SIGMA, + pynutil.delete("un "), + pynini.accep("[BOS]"), + pynini.project(higher_powers_of_ten, "output"), + NEMO_SIGMA, ) # we drop 'un' from these ordinals (millionths, not one-millionths) graph_higher_powers_of_ten = multiples_of_thousand | graph_hundreds | graph_higher_powers_of_ten @@ -82,10 +121,16 @@ def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = # Blocking the digits and hundreds from Cardinal graph graph_fractions_cardinals = pynini.cdrewrite( - block_three_to_ten | block_higher_powers_of_ten, pynini.accep("[BOS]"), pynini.accep("[EOS]"), NEMO_SIGMA, + block_three_to_ten | block_higher_powers_of_ten, + pynini.accep("[BOS]"), + pynini.accep("[EOS]"), + NEMO_SIGMA, ) graph_fractions_cardinals @= NEMO_CHAR.plus @ pynini.cdrewrite( - pynutil.delete("0"), pynini.accep("[BOS]"), pynini.accep("[EOS]"), NEMO_SIGMA, + pynutil.delete("0"), + pynini.accep("[BOS]"), + pynini.accep("[EOS]"), + NEMO_SIGMA, ) # Empty characters become '0' for NEMO_CHAR fst, so need to block graph_fractions_cardinals @= cardinal_graph graph_fractions_cardinals += pynutil.insert( diff --git a/nemo_text_processing/text_normalization/es/taggers/measure.py b/nemo_text_processing/text_normalization/es/taggers/measure.py index 2d1eb3ff4..a63677c47 100644 --- a/nemo_text_processing/text_normalization/es/taggers/measure.py +++ b/nemo_text_processing/text_normalization/es/taggers/measure.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_NON_BREAKING_SPACE, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil unit = pynini.string_file(get_abs_path("data/measures/measurements.tsv")) unit_complex = pynini.string_file(get_abs_path("data/measures/measurements_complex.tsv")) @@ -78,7 +79,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) optional_unit_denominator = pynini.closure( - pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, 0, 1, + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, ) complex_unit_singular_graph = ( diff --git a/nemo_text_processing/text_normalization/es/taggers/money.py b/nemo_text_processing/text_normalization/es/taggers/money.py index 3cb074550..bbb973465 100644 --- a/nemo_text_processing/text_normalization/es/taggers/money.py +++ b/nemo_text_processing/text_normalization/es/taggers/money.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -23,7 +25,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import decimal_separator from nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels -from pynini.lib import pynutil maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) diff --git a/nemo_text_processing/text_normalization/es/taggers/ordinal.py b/nemo_text_processing/text_normalization/es/taggers/ordinal.py index ca22e35b3..d1822103a 100644 --- a/nemo_text_processing/text_normalization/es/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/es/taggers/ordinal.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_SIGMA, @@ -21,7 +23,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import roman_to_int, strip_accent from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil digit = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/digit.tsv"))) teens = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/teen.tsv"))) @@ -48,7 +49,7 @@ def get_one_to_one_thousand(cardinal: 'pynini.FstLike') -> 'pynini.FstLike': class OrdinalFst(GraphFst): """ Finite state transducer for classifying ordinal - "21.º" -> ordinal { integer: "vigésimo primero" morphosyntactic_features: "gender_masc" } + "21.º" -> ordinal { integer: "vigésimo primero" morphosyntactic_features: "gender_masc" } This class converts ordinal up to the millionth (millonésimo) order (exclusive). This FST also records the ending of the ordinal (called "morphosyntactic_features"): @@ -131,7 +132,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): # Managing Romanization, excluding words that may be ambiguous roman_ordinals = roman_to_int(ordinal_graph) - exceptions = pynini.accep("vi") | pynini.accep("di") | pynini.accep("mi") + # exceptions = pynini.accep("vi") | pynini.accep("di") | pynini.accep("mi") + exceptions = pynini.string_file(get_abs_path("data/ordinals/roman_exceptions.tsv")) graph_exception = pynini.project(exceptions, 'input') roman_ordinals = (pynini.project(roman_ordinals, "input") - graph_exception.arcsort()) @ roman_ordinals diff --git a/nemo_text_processing/text_normalization/es/taggers/telephone.py b/nemo_text_processing/text_normalization/es/taggers/telephone.py index a1d95b468..1cc332f07 100644 --- a/nemo_text_processing/text_normalization/es/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/es/taggers/telephone.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_SIGMA, @@ -21,7 +23,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import ones from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) @@ -42,8 +43,8 @@ class TelephoneFst(GraphFst): (we ignore more complicated cases such as "doscientos y dos" or "tres nueves"). Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) """ def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/es/taggers/time.py b/nemo_text_processing/text_normalization/es/taggers/time.py index e5bebcfdc..de2752657 100644 --- a/nemo_text_processing/text_normalization/es/taggers/time.py +++ b/nemo_text_processing/text_normalization/es/taggers/time.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -20,7 +22,6 @@ insert_space, ) from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil time_zones = pynini.string_file(get_abs_path("data/time/time_zone.tsv")) suffix = pynini.string_file(get_abs_path("data/time/time_suffix.tsv")) @@ -114,7 +115,9 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): time_zone_graph = time_zones + pynini.closure(utc_or_gmt_diff, 0, 1) final_time_zone_optional = pynini.closure( - delete_space + insert_space + pynutil.insert("zone: \"") + time_zone_graph + pynutil.insert("\""), 0, 1, + delete_space + insert_space + pynutil.insert("zone: \"") + time_zone_graph + pynutil.insert("\""), + 0, + 1, ) # 02.30 h diff --git a/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py index 825daa50e..165f5eeca 100644 --- a/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/es/taggers/tokenize_and_classify.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_WHITE_SPACE, GraphFst, @@ -36,7 +37,7 @@ from nemo_text_processing.text_normalization.es.taggers.time import TimeFst from nemo_text_processing.text_normalization.es.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.es.taggers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -68,13 +69,14 @@ def __init__( os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" far_file = os.path.join( - cache_dir, f"_{input_case}_es_tn_{deterministic}_deterministic{whitelist_file}.far" + cache_dir, + f"_{input_case}_es_tn_{deterministic}_deterministic{whitelist_file}.far", ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") self.cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = self.cardinal.fst @@ -85,10 +87,17 @@ def __init__( self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) decimal_graph = self.decimal.fst - self.fraction = FractionFst(cardinal=self.cardinal, ordinal=self.ordinal, deterministic=deterministic) + self.fraction = FractionFst( + cardinal=self.cardinal, + ordinal=self.ordinal, + deterministic=deterministic, + ) fraction_graph = self.fraction.fst self.measure = MeasureFst( - cardinal=self.cardinal, decimal=self.decimal, fraction=self.fraction, deterministic=deterministic + cardinal=self.cardinal, + decimal=self.decimal, + fraction=self.fraction, + deterministic=deterministic, ) measure_graph = self.measure.fst self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic) @@ -100,7 +109,11 @@ def __init__( telephone_graph = self.telephone.fst self.electronic = ElectronicFst(deterministic=deterministic) electronic_graph = self.electronic.fst - self.money = MoneyFst(cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic) + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) money_graph = self.money.fst self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) whitelist_graph = self.whitelist.fst @@ -117,7 +130,7 @@ def __init__( | pynutil.add_weight(decimal_graph, 1.1) | pynutil.add_weight(money_graph, 1.09) | pynutil.add_weight(telephone_graph, 1.11) - | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.11) | pynutil.add_weight(word_graph, 200) ) punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") @@ -146,4 +159,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/es/taggers/whitelist.py b/nemo_text_processing/text_normalization/es/taggers/whitelist.py index 9f6c6d99b..ac591017b 100644 --- a/nemo_text_processing/text_normalization/es/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/es/taggers/whitelist.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/es/taggers/word.py b/nemo_text_processing/text_normalization/es/taggers/word.py index 90ecbc99c..cc2eeec2f 100644 --- a/nemo_text_processing/text_normalization/es/taggers/word.py +++ b/nemo_text_processing/text_normalization/es/taggers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/es/utils.py b/nemo_text_processing/text_normalization/es/utils.py index a357ba201..375bc1fb1 100644 --- a/nemo_text_processing/text_normalization/es/utils.py +++ b/nemo_text_processing/text_normalization/es/utils.py @@ -37,6 +37,6 @@ def load_labels(abs_path): Returns dictionary of mappings """ - label_tsv = open(abs_path) - labels = list(csv.reader(label_tsv, delimiter="\t")) + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) return labels diff --git a/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py index 1806d1477..968075e11 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/cardinal.py @@ -12,24 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from nemo_text_processing.text_normalization.es.graph_utils import ( add_cardinal_apocope_fem, shift_cardinal_gender, strip_cardinal_apocope, ) -from pynini.lib import pynutil class CardinalFst(GraphFst): """ - Finite state transducer for verbalizing cardinals - e.g. cardinal { integer: "dos" } -> "dos" + Finite state transducer for verbalizing cardinals + e.g. cardinal { integer: "dos" } -> "dos" - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/es/verbalizers/date.py b/nemo_text_processing/text_normalization/es/verbalizers/date.py index bf4393fee..586890d73 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/date.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -21,7 +23,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope from nemo_text_processing.text_normalization.es.taggers.date import articles -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/es/verbalizers/decimals.py b/nemo_text_processing/text_normalization/es/verbalizers/decimals.py index 643c9a5dd..4feedd37d 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/decimals.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/decimals.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -26,19 +28,18 @@ shift_number_gender, strip_cardinal_apocope, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. - decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones - decimal { integer_part: "un" quantity: "billón" } -> un billón + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones + decimal { integer_part: "un" quantity: "billón" } -> un billón Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="decimal", kind="classify", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/es/verbalizers/electronic.py b/nemo_text_processing/text_normalization/es/verbalizers/electronic.py index 0c866c550..ebef3fae3 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/electronic.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,15 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, + NEMO_SPACE, GraphFst, + colon, delete_preserve_order, - insert_space, + domain_string, + double_quotes, + protocol_string, + username_string, ) from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) @@ -28,6 +34,7 @@ graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) +arroba = "arroba" class ElectronicFst(GraphFst): @@ -44,34 +51,42 @@ class ElectronicFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) - graph_digit_no_zero = ( digit_no_zero @ pynini.cdrewrite(pynini.cross("un", "uno"), "", "", NEMO_SIGMA).optimize() ) graph_digit = graph_digit_no_zero | zero def add_space_after_char(): - return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + ( - NEMO_NOT_QUOTE - pynini.accep(" ") + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) ) verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) - user_name = pynutil.delete("username: \"") + add_space_after_char() + pynutil.delete("\"") + user_name = ( + pynutil.delete(username_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() + + pynutil.delete(double_quotes) + ) user_name @= verbalize_characters convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common - domain = convert_defaults + pynini.closure(insert_space + convert_defaults) + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) domain @= verbalize_characters - domain = pynutil.delete("domain: \"") + domain + pynutil.delete("\"") + domain = ( + pynutil.delete(domain_string + colon + NEMO_SPACE + double_quotes) + domain + pynutil.delete(double_quotes) + ) protocol = ( - pynutil.delete("protocol: \"") + pynutil.delete(protocol_string + colon + NEMO_SPACE + double_quotes) + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) - + pynutil.delete("\"") + + pynutil.delete(double_quotes) ) - self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | ( - user_name + pynini.accep(" ") + pynutil.insert("arroba ") + domain + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert(arroba + NEMO_SPACE) + domain + | (pynutil.insert(arroba + NEMO_SPACE) + user_name) ) + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/es/verbalizers/fraction.py b/nemo_text_processing/text_normalization/es/verbalizers/fraction.py index fdbf83e68..5d7afc1b7 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/fraction.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_NOT_QUOTE, @@ -27,20 +29,19 @@ shift_cardinal_gender, strip_cardinal_apocope, ) -from pynini.lib import pynutil class FractionFst(GraphFst): """ - Finite state transducer for verbalizing fraction - e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> - treinta y tres y cuatro quintos + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> + treinta y tres y cuatro quintos - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) @@ -140,7 +141,8 @@ def __init__(self, deterministic: bool = True): fraction_with_one_fem = numerator_one_fem + delete_space + insert_space fraction_with_one_fem += pynini.union( - denominator_singular_fem @ merge_stem, denominator_singular_fem @ merge_into_single_word, + denominator_singular_fem @ merge_stem, + denominator_singular_fem @ merge_into_single_word, ) # Both forms exists fraction_with_one_fem += pynutil.insert(" parte") fraction_with_one_fem @= pynini.cdrewrite( @@ -149,7 +151,8 @@ def __init__(self, deterministic: bool = True): fraction_default_fem = numerator_fem + delete_space + insert_space fraction_default_fem += pynini.union( - denominator_plural_fem @ merge_stem, denominator_plural_fem @ merge_into_single_word, + denominator_plural_fem @ merge_stem, + denominator_plural_fem @ merge_into_single_word, ) fraction_default_fem += pynutil.insert(" partes") diff --git a/nemo_text_processing/text_normalization/es/verbalizers/measure.py b/nemo_text_processing/text_normalization/es/verbalizers/measure.py index f9659acb7..de877446d 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/measure.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import ones from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv")) unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv")) diff --git a/nemo_text_processing/text_normalization/es/verbalizers/money.py b/nemo_text_processing/text_normalization/es/verbalizers/money.py index d2ba9c707..e83e0444d 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/money.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -21,7 +23,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import shift_cardinal_gender, strip_cardinal_apocope from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) diff --git a/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py index 009cdf343..039b00de5 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/ordinal.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, NEMO_SPACE, GraphFst from nemo_text_processing.text_normalization.es.graph_utils import shift_number_gender -from pynini.lib import pynutil class OrdinalFst(GraphFst): @@ -22,7 +23,7 @@ class OrdinalFst(GraphFst): Finite state transducer for verbalizing ordinals e.g. ordinal { integer: "tercer" } } -> "tercero" -> "tercera" - -> "tercer" + -> "tercer" Args: deterministic: if True will provide a single transduction option, diff --git a/nemo_text_processing/text_normalization/es/verbalizers/telephone.py b/nemo_text_processing/text_normalization/es/verbalizers/telephone.py index 9f0b4fcc2..e54fa0d95 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/telephone.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/telephone.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + class TelephoneFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/es/verbalizers/time.py b/nemo_text_processing/text_normalization/es/verbalizers/time.py index 23fa1d180..7ca9b43a5 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/time.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -21,7 +23,6 @@ insert_space, ) from nemo_text_processing.text_normalization.es.utils import get_abs_path -from pynini.lib import pynutil alt_minutes = pynini.string_file(get_abs_path("data/time/alt_minutes.tsv")) diff --git a/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py index 9a7958e23..d2066a1bd 100644 --- a/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/es/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -24,7 +25,7 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.es.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): @@ -47,7 +48,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"es_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst @@ -67,4 +68,3 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/fr/data/dates/__init__.py b/nemo_text_processing/text_normalization/fr/data/dates/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/fr/data/dates/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv b/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv new file mode 100644 index 000000000..6127bea93 --- /dev/null +++ b/nemo_text_processing/text_normalization/fr/data/dates/eras.tsv @@ -0,0 +1,8 @@ +20s twenties +30s thirties +40s forties +50s fifties +60s sixties +70s seventies +80s eighties +90s nineties \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/fr/data/dates/months.tsv b/nemo_text_processing/text_normalization/fr/data/dates/months.tsv new file mode 100644 index 000000000..98a4e7d5d --- /dev/null +++ b/nemo_text_processing/text_normalization/fr/data/dates/months.tsv @@ -0,0 +1,12 @@ +1 janvier +2 février +3 mars +4 avril +5 mai +6 juin +7 juillet +8 août +9 septembre +10 octobre +11 novembre +12 décembre \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/it/data/__init__ .py b/nemo_text_processing/text_normalization/fr/data/measures/__init__.py similarity index 100% rename from nemo_text_processing/text_normalization/it/data/__init__ .py rename to nemo_text_processing/text_normalization/fr/data/measures/__init__.py diff --git a/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv b/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv new file mode 100644 index 000000000..55f0d4643 --- /dev/null +++ b/nemo_text_processing/text_normalization/fr/data/measures/measurements.tsv @@ -0,0 +1,17 @@ +m mètres +m² mètres carrés +m³ mètres cubes +s secondes +min minutes +h heures +° degrés +°C degrés celsius +g grammes +l litres +kg kilos +'' pouce +lb livres +% pour cent +‰ pour mille +km/h kilomètres heure +m/h mètres à l’heure \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/fr/data/whitelist.tsv b/nemo_text_processing/text_normalization/fr/data/whitelist.tsv index 80bc9f6d4..dc563bdab 100644 --- a/nemo_text_processing/text_normalization/fr/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/fr/data/whitelist.tsv @@ -1 +1,13 @@ -bonjour \ No newline at end of file +Mᵐᵉ madame +Mᵐᵉˢ mesdames +Mˡˡᵉ mademoiselle +Mˡˡᵉˢ mademoiselles +Dʳ docteur +Dʳˢ docteurs +Dʳᵉ docteure +Dʳᵉˢ docteures +apr. J.-C. après jésus-christ +av. J.-C. avant Jésus-Christ +le hon. l’honorable +le très hon. le très hononrable +% pour cent \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/fr/taggers/cardinal.py b/nemo_text_processing/text_normalization/fr/taggers/cardinal.py index b95f6466a..49fda53ed 100644 --- a/nemo_text_processing/text_normalization/fr/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/fr/taggers/cardinal.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space from nemo_text_processing.text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class CardinalFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/fr/taggers/date.py b/nemo_text_processing/text_normalization/fr/taggers/date.py new file mode 100644 index 000000000..3c96f6913 --- /dev/null +++ b/nemo_text_processing/text_normalization/fr/taggers/date.py @@ -0,0 +1,97 @@ +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst +from nemo_text_processing.text_normalization.fr.utils import get_abs_path + +# TODO: add articles? 'le...' + +month_numbers = pynini.string_file(get_abs_path("data/dates/months.tsv")) +eras = pynini.string_file(get_abs_path("data/dates/eras.tsv")) +delete_leading_zero = ( + pynutil.delete("0") | (NEMO_DIGIT - "0") +) + NEMO_DIGIT # reminder, NEMO_DIGIT = filter on digits + + +class DateFst(GraphFst): + '''Finite state transducer for classyfing dates, e.g.: + '02.03.2003' -> date {day: 'deux' month: 'mai' year: 'deux mille trois' preserve order: true} + ''' + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="dates", kind="classify") + + cardinal_graph = cardinal.all_nums_no_tokens + + # 'le' -> 'le', 'les' -> 'les' + le_determiner = pynini.accep("le ") | pynini.accep("les ") + self.optional_le = pynini.closure(le_determiner, 0, 1) + + # '01' -> 'un' + optional_leading_zero = delete_leading_zero | NEMO_DIGIT + valid_day_number = pynini.union(*[str(x) for x in range(1, 32)]) + premier = pynini.string_map([("1", "premier")]) + day_number_to_word = premier | cardinal_graph + + digit_to_day = self.optional_le + optional_leading_zero @ valid_day_number @ day_number_to_word + self.day_graph = pynutil.insert("day: \"") + digit_to_day + pynutil.insert("\"") + + # '03' -> 'mars' + normalize_month_number = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)]) + number_to_month = month_numbers.optimize() + month_graph = normalize_month_number @ number_to_month + self.month_graph = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + + # 2025 -> deux mille vingt cinq + accept_year_digits = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 1, 3) + digits_to_year = accept_year_digits @ cardinal_graph + self.year_graph = pynutil.insert("year: \"") + digits_to_year + pynutil.insert("\"") + + # Putting it all together + self.fst = pynini.accep("") + + for separator in ["/", ".", "-"]: + self.fst |= ( + pynutil.insert("date { ") + + self.day_graph + + pynutil.delete(separator) + + pynutil.insert(" ") + + self.month_graph + + pynini.closure(pynutil.delete(separator) + pynutil.insert(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true }") + ) + + # Accepts "janvier", "février", etc + month_name_graph = pynutil.insert("month: \"") + month_numbers.project("output") + pynutil.insert("\"") + + self.fst |= ( + pynutil.insert("date { ") + + self.day_graph + + pynini.accep(" ") + + month_name_graph + + pynini.closure(pynini.accep(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true}") + ) + + # Accepts "70s", "80s", etc + self.fst |= pynutil.insert("date { year: \"") + eras + pynutil.insert("\" preserve_order: true }") + + # Accepts date ranges, "17-18-19 juin" -> date { day: "17" day: "18": day: "19"} + for separator in ["-", "/"]: + day_range_graph = ( + pynutil.insert("day: \"") + + pynini.closure(digit_to_day + pynutil.delete(separator) + pynutil.insert(" "), 1) + + digit_to_day + + pynutil.insert("\"") + ) + + self.fst |= ( + pynutil.insert("date { ") + + day_range_graph + + pynini.accep(" ") + + month_name_graph + + pynini.closure(pynini.accep(" ") + self.year_graph, 0, 1) + + pynutil.insert(" preserve_order: true }") + ) + + self.fst = self.fst.optimize() diff --git a/nemo_text_processing/text_normalization/fr/taggers/decimals.py b/nemo_text_processing/text_normalization/fr/taggers/decimals.py index 49bf55037..a6e492e01 100644 --- a/nemo_text_processing/text_normalization/fr/taggers/decimals.py +++ b/nemo_text_processing/text_normalization/fr/taggers/decimals.py @@ -14,6 +14,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -22,7 +24,6 @@ insert_space, ) from nemo_text_processing.text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) diff --git a/nemo_text_processing/text_normalization/fr/taggers/fraction.py b/nemo_text_processing/text_normalization/fr/taggers/fraction.py index f65ee87ed..a8377bf6f 100644 --- a/nemo_text_processing/text_normalization/fr/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/fr/taggers/fraction.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/fr/taggers/ordinal.py b/nemo_text_processing/text_normalization/fr/taggers/ordinal.py index 4c8c8c73d..73b42053c 100644 --- a/nemo_text_processing/text_normalization/fr/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/fr/taggers/ordinal.py @@ -14,15 +14,16 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SPACE, GraphFst from nemo_text_processing.text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): """ Finite state transducer for classifying ordinal - "2e" -> ordinal { integer: "deux" morphosyntactic_features: "ième" } + "2e" -> ordinal { integer: "deux" morphosyntactic_features: "ième" } This grammar covers from single digits to hundreds of billions ("milliardième" in French). This FST also records the ending of the ordinal (called "morphosyntactic_features"). Args: diff --git a/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py index 55f219d2a..b3e10a4ec 100644 --- a/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/fr/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_WHITE_SPACE, GraphFst, @@ -25,12 +26,13 @@ ) from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst from nemo_text_processing.text_normalization.fr.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.fr.taggers.date import DateFst from nemo_text_processing.text_normalization.fr.taggers.decimals import DecimalFst from nemo_text_processing.text_normalization.fr.taggers.fraction import FractionFst from nemo_text_processing.text_normalization.fr.taggers.ordinal import OrdinalFst from nemo_text_processing.text_normalization.fr.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.fr.taggers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -61,13 +63,14 @@ def __init__( os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" far_file = os.path.join( - cache_dir, f"_{input_case}_fr_tn_{deterministic}_deterministic{whitelist_file}.far" + cache_dir, + f"_{input_case}_fr_tn_{deterministic}_deterministic{whitelist_file}.far", ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") self.cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = self.cardinal.fst @@ -78,15 +81,23 @@ def __init__( self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) decimal_graph = self.decimal.fst - self.fraction = FractionFst(cardinal=self.cardinal, ordinal=self.ordinal, deterministic=deterministic) + self.fraction = FractionFst( + cardinal=self.cardinal, + ordinal=self.ordinal, + deterministic=deterministic, + ) fraction_graph = self.fraction.fst word_graph = WordFst(deterministic=deterministic).fst self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) whitelist_graph = self.whitelist.fst punct_graph = PunctuationFst(deterministic=deterministic).fst + self.date = DateFst(self.cardinal, deterministic=deterministic) + date_graph = self.date.fst + classify = ( pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(date_graph, 1.1) | pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(fraction_graph, 1.09) | pynutil.add_weight(ordinal_graph, 1.1) @@ -104,14 +115,7 @@ def __init__( pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) ) - graph = token_plus_punct + pynini.closure( - ( - pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) - | (pynutil.insert(" ") + punct + pynutil.insert(" ")) - ) - + token_plus_punct - ) - + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) graph = delete_space + graph + delete_space graph |= punct @@ -119,4 +123,4 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/fr/taggers/whitelist.py b/nemo_text_processing/text_normalization/fr/taggers/whitelist.py index d84cc7f02..06dde93cd 100644 --- a/nemo_text_processing/text_normalization/fr/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/fr/taggers/whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.fr.utils import get_abs_path, load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/fr/taggers/word.py b/nemo_text_processing/text_normalization/fr/taggers/word.py index a67d9e9a3..c49a6ffaf 100644 --- a/nemo_text_processing/text_normalization/fr/taggers/word.py +++ b/nemo_text_processing/text_normalization/fr/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/fr/utils.py b/nemo_text_processing/text_normalization/fr/utils.py index 4f6882b51..7523e5762 100644 --- a/nemo_text_processing/text_normalization/fr/utils.py +++ b/nemo_text_processing/text_normalization/fr/utils.py @@ -22,7 +22,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path @@ -34,7 +34,7 @@ def load_labels(abs_path): Args: abs_path: absolute path - + Returns dictionary of mappings """ label_tsv = open(abs_path) diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py b/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py index a1cf281f0..d6c90c955 100644 --- a/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py +++ b/nemo_text_processing/text_normalization/fr/verbalizers/__init__.py @@ -1,13 +1,13 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py index a911d3430..347922a1d 100644 --- a/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/fr/verbalizers/cardinal.py @@ -12,18 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space + class CardinalFst(GraphFst): """ - Finite state transducer for verbalizing cardinals - e.g. cardinal { negative: "true" integer: "un milliard et un" } -> "moins un milliard et un" - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + Finite state transducer for verbalizing cardinals + e.g. cardinal { negative: "true" integer: "un milliard et un" } -> "moins un milliard et un" + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/date.py b/nemo_text_processing/text_normalization/fr/verbalizers/date.py new file mode 100644 index 000000000..8c8c1aa21 --- /dev/null +++ b/nemo_text_processing/text_normalization/fr/verbalizers/date.py @@ -0,0 +1,51 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date {day: "deux" month: "mars" year: "deux mille trois" preserve_order: true} -> deux mars deux mille trois + Args: + ordinal: OrdinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + day = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + month = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + year = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + decade = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + graph_dmy = day + NEMO_SPACE + month + pynini.closure(NEMO_SPACE + year, 0, 1) + delete_preserve_order + graph_my = month + NEMO_SPACE + year + delete_preserve_order + graph_decade = decade + delete_preserve_order + + self.graph = graph_dmy | graph_my | graph_decade + + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py b/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py index a248c2c1a..a720b405b 100644 --- a/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py +++ b/nemo_text_processing/text_normalization/fr/verbalizers/decimals.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -19,18 +21,17 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. - decimal { negative: "true" integer_part: "onze" fractional_part: "quatre cent six" quantity: "millions" preserve_order: true } -> moins onze virgule quatre cent six millions - decimal { integer_part: "cent quatorze" quantity: "billions" preserve_order: true } -> cent quatorze billions + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "onze" fractional_part: "quatre cent six" quantity: "millions" preserve_order: true } -> moins onze virgule quatre cent six millions + decimal { integer_part: "cent quatorze" quantity: "billions" preserve_order: true } -> cent quatorze billions Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="decimal", kind="classify", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py b/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py index 8032f80bc..9388cf343 100644 --- a/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/fr/verbalizers/fraction.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,18 +22,17 @@ insert_space, ) from nemo_text_processing.text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class FractionFst(GraphFst): """ - Finite state transducer for verbalizing fraction - e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> - treinta y tres y cuatro quintos - Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + Finite state transducer for verbalizing fraction + e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } -> + treinta y tres y cuatro quintos + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, ordinal: GraphFst, deterministic: bool = True): super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py index 039d5a12f..9e0233f6e 100644 --- a/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/fr/verbalizers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst from nemo_text_processing.text_normalization.fr.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py index 02510ea5f..3ea0117af 100644 --- a/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/fr/verbalizers/verbalize.py @@ -14,6 +14,7 @@ from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.fr.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.fr.verbalizers.date import DateFst from nemo_text_processing.text_normalization.fr.verbalizers.decimals import DecimalFst from nemo_text_processing.text_normalization.fr.verbalizers.fraction import FractionFst from nemo_text_processing.text_normalization.fr.verbalizers.ordinal import OrdinalFst @@ -40,6 +41,8 @@ def __init__(self, deterministic: bool = True): fraction = FractionFst(ordinal=ordinal, deterministic=deterministic) fraction_graph = fraction.fst whitelist_graph = WhiteListFst(deterministic=deterministic).fst + date = DateFst(deterministic=deterministic) + date_graph = date.fst - graph = cardinal_graph | decimal_graph | ordinal_graph | fraction_graph | whitelist_graph + graph = cardinal_graph | decimal_graph | ordinal_graph | fraction_graph | whitelist_graph | date_graph self.fst = graph diff --git a/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py index c289d016c..0313f7f5b 100644 --- a/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/fr/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -24,7 +25,7 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.fr.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): @@ -46,7 +47,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"fr_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst @@ -66,4 +67,4 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") + logger.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/hi/__init__.py b/nemo_text_processing/text_normalization/hi/__init__.py new file mode 100644 index 000000000..6cd1f01f4 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.hi.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.hi.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.hi.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/hi/data/__init__.py b/nemo_text_processing/text_normalization/hi/data/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/address/__init__.py b/nemo_text_processing/text_normalization/hi/data/address/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/address/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/address/cities.tsv b/nemo_text_processing/text_normalization/hi/data/address/cities.tsv new file mode 100644 index 000000000..0199bf0cb --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/address/cities.tsv @@ -0,0 +1,36 @@ +अमरावती +ईटानगर +दिसपुर +पटना +रायपुर +पणजी +गांधीनगर +चंडीगढ़ +शिमला +रांची +बेंगलुरु +तिरुवनंतपुरम +भोपाल +मुंबई +इम्फाल +शिलांग +आइजोल +कोहिमा +भुवनेश्वर +जयपुर +गंगटोक +चेन्नई +हैदराबाद +अगरतला +लखनऊ +देहरादून +कोलकाता +पोर्ट ब्लेयर +दमन +नई दिल्ली +श्रीनगर +जम्मू +लेह +कारगिल +कवरत्ती +पुडुचेरी diff --git a/nemo_text_processing/text_normalization/hi/data/address/context.tsv b/nemo_text_processing/text_normalization/hi/data/address/context.tsv new file mode 100644 index 000000000..d57bfd7d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/address/context.tsv @@ -0,0 +1,47 @@ +हाउस +प्लॉट +बूथ +अपार्टमेंट +फ्लैट +यूनिट +टावर +कॉम्प्लेक्स +मंजिल +फ्लोर +ब्लॉक +सेक्टर +फेज +रोड +सड़क +मार्ग +स्ट्रीट +गली +राजमार्ग +ड्राइव +डिस्ट्रिक्ट +बाईपास +हाइवे +पार्कवे +कॉलोनी +नगर +पार्क +एस्टेट +बोलवार्ड +मार्केट +सेंटर +पिन +गांव +पास +ब्रिगेड +नियर +स्क्वेर +मॉल +टॉवर +इंस्टीट्यूट +पिलर +मेट्रो +एवेन्यू +वेस्ट +सामने +पीछे +वीया \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/address/letters.tsv b/nemo_text_processing/text_normalization/hi/data/address/letters.tsv new file mode 100644 index 000000000..77bc608be --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/address/letters.tsv @@ -0,0 +1,26 @@ +a ए +b बी +c सी +d डी +e ई +f एफ +g जी +h एच +i आई +j जे +k के +l एल +m एम +n एन +o ओ +p पी +q क्यू +r आर +s एस +t टी +u यू +v वी +w डब्ल्यू +x एक्स +y वाई +z ज़ेड \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/address/special_characters.tsv b/nemo_text_processing/text_normalization/hi/data/address/special_characters.tsv new file mode 100644 index 000000000..ca5b068bd --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/address/special_characters.tsv @@ -0,0 +1,2 @@ +- हाइफ़न +/ बटा \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/address/states.tsv b/nemo_text_processing/text_normalization/hi/data/address/states.tsv new file mode 100644 index 000000000..1e2b6c358 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/address/states.tsv @@ -0,0 +1,36 @@ +आंध्र प्रदेश +अरुणाचल प्रदेश +असम +बिहार +छत्तीसगढ़ +गोवा +गुजरात +हरियाणा +हिमाचल प्रदेश +झारखंड +कर्नाटक +केरल +मध्य प्रदेश +महाराष्ट्र +मणिपुर +मेघालय +मिज़ोरम +नागालैंड +ओडिशा +पंजाब +राजस्थान +सिक्किम +तमिलनाडु +तेलंगाना +त्रिपुरा +उत्तर प्रदेश +उत्तराखंड +पश्चिम बंगाल +अंडमान और निकोबार द्वीप समूह +चंडीगढ़ +दादरा और नगर हवेली और दमन और दीव +दिल्ली +जम्मू और कश्मीर +लद्दाख +लक्षद्वीप +पुडुचेरी diff --git a/nemo_text_processing/text_normalization/hi/data/date/__init__.py b/nemo_text_processing/text_normalization/hi/data/date/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/date/days.tsv b/nemo_text_processing/text_normalization/hi/data/date/days.tsv new file mode 100644 index 000000000..6df0fa3d4 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/date/days.tsv @@ -0,0 +1,62 @@ +01 एक +02 दो +03 तीन +04 चार +05 पाँच +06 छह +07 सात +08 आठ +09 नौ +10 दस +11 ग्यारह +12 बारह +13 तेरह +14 चौदह +15 पंद्रह +16 सोलह +17 सत्रह +18 अठारह +19 उन्नीस +20 बीस +21 इक्कीस +22 बाईस +23 तेईस +24 चौबीस +25 पच्चीस +26 छब्बीस +27 सत्ताईस +28 अट्ठाईस +29 उनतीस +30 तीस +31 इकतीस +०१ एक +०२ दो +०३ तीन +०४ चार +०५ पाँच +०६ छह +०७ सात +०८ आठ +०९ नौ +१० दस +११ ग्यारह +१२ बारह +१३ तेरह +१४ चौदह +१५ पंद्रह +१६ सोलह +१७ सत्रह +१८ अठारह +१९ उन्नीस +२० बीस +२१ इक्कीस +२२ बाईस +२३ तेईस +२४ चौबीस +२५ पच्चीस +२६ छब्बीस +२७ सत्ताईस +२८ अट्ठाईस +२९ उनतीस +३० तीस +३१ इकतीस \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/date/months.tsv b/nemo_text_processing/text_normalization/hi/data/date/months.tsv new file mode 100644 index 000000000..3667f07cf --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/date/months.tsv @@ -0,0 +1,24 @@ +01 जनवरी +02 फरवरी +03 मार्च +04 अप्रैल +05 मई +06 जून +07 जुलाई +08 अगस्त +09 सितंबर +10 अक्टूबर +11 नवंबर +12 दिसंबर +०१ जनवरी +०२ फरवरी +०३ मार्च +०४ अप्रैल +०५ मई +०६ जून +०७ जुलाई +०८ अगस्त +०९ सितंबर +१० अक्टूबर +११ नवंबर +१२ दिसंबर \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/date/prefixes.tsv b/nemo_text_processing/text_normalization/hi/data/date/prefixes.tsv new file mode 100644 index 000000000..6166ec327 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/date/prefixes.tsv @@ -0,0 +1,4 @@ +सन् +सन +साल +दशक \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/date/suffixes.tsv b/nemo_text_processing/text_normalization/hi/data/date/suffixes.tsv new file mode 100644 index 000000000..6806d3f12 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/date/suffixes.tsv @@ -0,0 +1,10 @@ + में + का + की + के + से + तक + ईस्वी + शताब्दी + दशक + सदी \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/date/year_suffix.tsv b/nemo_text_processing/text_normalization/hi/data/date/year_suffix.tsv new file mode 100644 index 000000000..acb37d534 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/date/year_suffix.tsv @@ -0,0 +1,2 @@ +ई. पू. ईसा पूर्व +ई. ईसवी \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/__init__.py b/nemo_text_processing/text_normalization/hi/data/electronic/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/domain.tsv b/nemo_text_processing/text_normalization/hi/data/electronic/domain.tsv new file mode 100644 index 000000000..dccb5dd90 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/domain.tsv @@ -0,0 +1,24 @@ +com +org +net +edu +gov +in +co +io +ai +uk +us +au +ca +ac +res +nic +ernet +tv +me +tech +dev +app +biz +info diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/elements.tsv b/nemo_text_processing/text_normalization/hi/data/electronic/elements.tsv new file mode 100644 index 000000000..be4610634 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/elements.tsv @@ -0,0 +1,132 @@ +Ac +Ag +Al +Am +An +Ar +As +At +Au +B +Ba +Be +Bh +Bi +Bk +Br +Bu +Bz +C +Ca +Cd +Ce +Cf +Cl +Cm +Cn +Co +Cp +Cr +Cs +Cu +D +Db +Ds +Dy +En +Er +Es +Et +Eu +F +Fe +Fl +Fm +Fr +Ga +Gd +Ge +H +He +Hf +Hg +Ho +Hs +I +In +Ir +K +Kr +La +Li +Ln +Lr +Lu +Lv +M +Mc +Md +Me +Mg +Mn +Mo +Mt +N +Na +Nb +Nd +Ne +Nh +Ni +No +Np +O +Og +Os +P +Pa +Pb +Pd +Ph +Pm +Po +Pr +Pt +Pu +R +Ra +Rb +Re +Rf +Rg +Rh +Rn +Ru +S +Sb +Sc +Se +Sg +Si +Sm +Sn +Sr +T +Ta +Tb +Tc +Te +Th +Ti +Tl +Tm +Ts +U +V +W +X +Xe +Y +Yb +Zn +Zr \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/file_extensions.tsv b/nemo_text_processing/text_normalization/hi/data/electronic/file_extensions.tsv new file mode 100644 index 000000000..7283febf5 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/file_extensions.tsv @@ -0,0 +1,34 @@ +jpg +jpeg +png +gif +pdf +doc +docx +xls +xlsx +ppt +pptx +csv +txt +html +xml +json +css +js +py +java +cpp +zip +rar +tar +mp3 +mp4 +avi +mkv +mov +wav +svg +apk +exe +sql \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/protocols.tsv b/nemo_text_processing/text_normalization/hi/data/electronic/protocols.tsv new file mode 100644 index 000000000..dd897a4b6 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/protocols.tsv @@ -0,0 +1,5 @@ +https https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश +http http कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश +www www डॉट +httpswww https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश www डॉट +httpwww http कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश www डॉट diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/subscript_digit.tsv b/nemo_text_processing/text_normalization/hi/data/electronic/subscript_digit.tsv new file mode 100644 index 000000000..218fb5b6e --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/subscript_digit.tsv @@ -0,0 +1,10 @@ +₀ शून्य +₁ एक +₂ दो +₃ तीन +₄ चार +₅ पाँच +₆ छह +₇ सात +₈ आठ +₉ नौ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/symbol_classes.tsv b/nemo_text_processing/text_normalization/hi/data/electronic/symbol_classes.tsv new file mode 100644 index 000000000..cf17c8756 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/symbol_classes.tsv @@ -0,0 +1,16 @@ +. email,url,unix,windows +- email,url,unix,windows,chem +_ email,url,unix,windows +/ url,unix +$ unix +\ windows +( windows,chem +) windows,chem ++ url,chem +– chem +# url +? url +& url += url +% url +: url,windows \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/electronic/symbols.tsv b/nemo_text_processing/text_normalization/hi/data/electronic/symbols.tsv new file mode 100644 index 000000000..e720f5338 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/electronic/symbols.tsv @@ -0,0 +1,36 @@ +. डॉट +- हाइफ़न +_ अंडर स्कोर +@ एट +/ फॉरवर्ड स्लैश +\\ बैकवर्ड स्लैश +: कोलन +\# हैशटैग +$ डॉलर +% प्रतिशत +& एंड ++ प्लस += इक्वल +? क्वेश्चन मार्क +* स्टार +! एक्सक्लेमेशन मार्क +~ टिल्डा +^ कैरेट +| पाइप +` बैकटिक +; सेमीकोलन +, कॉमा +( ओपन ब्रेकेट +) क्लोज़ ब्रेकेट + स्पेस +' सिंगल कोट +" डबल कोट +< लेस दैन +> ग्रेटर दैन +\[ ओपन स्क्वेर ब्रेकेट +\] क्लोज़ स्क्वेर ब्रेकेट +{ ओपन कर्ली ब्रेकेट +} क्लोज़ कर्ली ब्रेकेट +– माइनस +⁻ माइनस +⁺ प्लस \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/fraction/__init__.py b/nemo_text_processing/text_normalization/hi/data/fraction/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/fraction/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/fraction/common_fractions.tsv b/nemo_text_processing/text_normalization/hi/data/fraction/common_fractions.tsv new file mode 100644 index 000000000..5e44cb502 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/fraction/common_fractions.tsv @@ -0,0 +1,10 @@ +१/२ आधा +१/३ तिहाई +२/३ दो तिहाई +१/४ चौथाई +३/४ तीन चौथाई +1/2 आधा +1/3 तिहाई +2/3 दो तिहाई +1/4 चौथाई +3/4 तीन चौथाई \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/measure/__init__.py b/nemo_text_processing/text_normalization/hi/data/measure/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_list.tsv b/nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_list.tsv new file mode 100644 index 000000000..6fcfb8b3a --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_list.tsv @@ -0,0 +1,5 @@ +हफ़्ते +सप्ताह +सदियां +सदियों + diff --git a/nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_map.tsv b/nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_map.tsv new file mode 100644 index 000000000..e190a80ef --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/quarterly_units_map.tsv @@ -0,0 +1,10 @@ +s सेकंड +hr घंटा +h घंटे +min मिनट +doz दर्जन +yr साल +hp हॉर्सपॉवर +d दिन +month महीना +months महीने diff --git a/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv new file mode 100644 index 000000000..d236dd51b --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/unit.tsv @@ -0,0 +1,145 @@ +°C डिग्री सेल्सियस +°F डिग्री फारेनहाइट +g ग्राम +kg किलोग्राम +mg मिलीग्राम +cg सेंटीग्राम +dg डेसीग्राम +Tg टेराग्राम +Mg मेगाग्राम +Gg गीगाग्राम +hg हेक्टोग्राम +dag डेकाग्राम +lb पाउंड +oz आउन्स +t टन +st स्टोन +q क्विंटल +m मीटर +cm सेंटीमीटर +mm मिलीमीटर +km किलोमीटर +dm डेसीमीटर +dam डेकामीटर +nm नैनोमीटर +hm हेक्टोमीटर +Mm मेगामीटर +my मिरियामीटर +mi मील +ft फीट +in इंच +yd यार्ड +µm माइक्रोमीटर +m² वर्ग मीटर +cm² वर्ग सेंटीमीटर +mm² वर्ग मिलीमीटर +km² वर्ग किलोमीटर +hm² वर्ग हेक्टोमीटर +dm² वर्ग डेसीमीटर +dam² वर्ग डेकामीटर +yd² वर्ग यार्ड +ft² वर्ग फीट +ac² वर्ग एकड़ +my² वर्ग मिरियामीटर +mi² वर्ग मील +nm² वर्ग नैनोमीटर +µm² वर्ग माइक्रोमीटर +in² वर्ग इंच +my³ घन मिरियामीटर +mi³ घन मील +ac³ घन एकड़ +ha हेक्टेयर +ac एकड़ +गज गज +गज² वर्ग गज +गज³ घन गज +m³ घन मीटर +cm³ घन सेंटीमीटर +mm³ घन मिलीमीटर +dam³ घन डेकामीटर +µm³ घन माइक्रोमीटर +ml³ घन मिलीलीटर +l लीटर +kl किलोलीटर +ml मिलीलीटर +ml² वर्ग मिलीलीटर +dl डेसीलीटर +hl हेक्टोलीटर +cl सेंटीलीटर +dal डेकालीटर +dl² वर्ग डेसीलीटर +dal² वर्ग डेकालीटर +dl³ घन डेसीलीटर +dal³ घन डेकालीटर +kL किलोलीटर +mL मिलीलीटर +mL² वर्ग मिलीलीटर +mL³ घन मिलीलीटर +dL डेसीलीटर +hL हेक्टोलीटर +cL सेंटीलीटर +daL डेकालीटर +dL² वर्ग डेसीलीटर +daL² वर्ग डेकालीटर +dL³ घन डेसीलीटर +daL³ घन डेकालीटर +GB गीगाबाइट +in³ घन इंच +ft³ घन फीट +yd³ घन यार्ड +my³ घन मिरियामीटर +mi³ घन मील +dm³ घन डेसीमीटर +dm³ घन डेसीमीटर +km³ घन किलोमीटर +nm³ घन नैनोमीटर +mm³ घन मिलीमीटर +qt क्वार्ट +gal गैलन +pt पिंट +MW मेगावाट +KW किलोवाट +b बिट +Mb मेगाबिट +GB गीगाबाइट +KB किलोबाइट +TB टेराबाइट +MB मेगाबाइट +PB पेटाबाइट +EB एक्साबाइट +ZB जेटाबाइट +YB योटाबाइट +BB ब्रोन्टोबाइट +Pa पास्कल +s सेकंड +hr घंटा +h घंटे +min मिनट +ha हेक्टेयर +ha² वर्ग हेक्टेयर +Ω ओम +MΩ मेगाओम +doz दर्जन +Hz हर्ट्ज़ +GHz गीगाहर्ट्ज़ +KHz किलोहर्ट्ज़ +dB डेसीबल +yr साल +hp हॉर्सपॉवर +d दिन +month महीना +months महीने +ct कैरेट +pH पीएच +km/h किलोमीटर प्रति घंटा +km/hr किलोमीटर प्रति घंटा +km/min किलोमीटर प्रति मिनट +m/h मीटर प्रति घंटा +m/hr मीटर प्रति घंटा +mi/s मील प्रति सेकंड +mi/h मील प्रति घंटा +mi/hr मील प्रति घंटा +mi/min मील प्रति मिनट +₹/ac रुपए प्रति एकड़ +x बाई +* बाई \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/measure/unit_year_formal.tsv b/nemo_text_processing/text_normalization/hi/data/measure/unit_year_formal.tsv new file mode 100644 index 000000000..a3c7b2162 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/unit_year_formal.tsv @@ -0,0 +1 @@ +yr वर्ष diff --git a/nemo_text_processing/text_normalization/hi/data/money/__init__.py b/nemo_text_processing/text_normalization/hi/data/money/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/money/currency.tsv b/nemo_text_processing/text_normalization/hi/data/money/currency.tsv new file mode 100644 index 000000000..8f4a955cc --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/money/currency.tsv @@ -0,0 +1,9 @@ +₹ रुपए +£ पाउंड +₩ वॉन +$ डॉलर +₺ लीरा +৳ टका +¥ येन +₦ नाइरा +€ यूरो \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/money/currency_singular.tsv b/nemo_text_processing/text_normalization/hi/data/money/currency_singular.tsv new file mode 100644 index 000000000..af8d793f2 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/money/currency_singular.tsv @@ -0,0 +1,9 @@ +₹ रुपया +£ पाउंड +₩ वॉन +$ डॉलर +₺ लीरा +৳ टका +¥ येन +₦ नाइरा +€ यूरो \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/money/major_minor_currencies.tsv b/nemo_text_processing/text_normalization/hi/data/money/major_minor_currencies.tsv new file mode 100644 index 000000000..a9186acc3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/money/major_minor_currencies.tsv @@ -0,0 +1,10 @@ +रुपए पैसे +रुपया पैसे +पाउंड पेंस +वॉन जिओन +डॉलर सेंट +लीरा कुरस +टका पैसे +येन सेन +नाइरा कोबो +यूरो सेंट \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/numbers/__init__.py b/nemo_text_processing/text_normalization/hi/data/numbers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/hi/data/numbers/digit.tsv new file mode 100644 index 000000000..6bacb7fc3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/numbers/digit.tsv @@ -0,0 +1,18 @@ +१ एक +२ दो +३ तीन +४ चार +५ पाँच +६ छह +७ सात +८ आठ +९ नौ +1 एक +2 दो +3 तीन +4 चार +5 पाँच +6 छह +7 सात +8 आठ +9 नौ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/numbers/hundred.tsv b/nemo_text_processing/text_normalization/hi/data/numbers/hundred.tsv new file mode 100644 index 000000000..049f029bc --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/numbers/hundred.tsv @@ -0,0 +1 @@ +१०० एक सौ diff --git a/nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties.tsv b/nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..fbf248266 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties.tsv @@ -0,0 +1,90 @@ +१० दस +११ ग्यारह +१२ बारह +१३ तेरह +१४ चौदह +१५ पंद्रह +१६ सोलह +१७ सत्रह +१८ अठारह +१९ उन्नीस +२० बीस +२१ इक्कीस +२२ बाईस +२३ तेईस +२४ चौबीस +२५ पच्चीस +२६ छब्बीस +२७ सत्ताईस +२८ अट्ठाईस +२९ उनतीस +३० तीस +३१ इकतीस +३२ बत्तीस +३३ तैंतीस +३४ चौंतीस +३५ पैंतीस +३६ छत्तीस +३७ सैंतीस +३८ अड़तीस +३९ उनतालीस +४० चालीस +४१ इकतालीस +४२ बयालीस +४३ तैंतालीस +४४ चौवालीस +४५ पैंतालीस +४६ छियालीस +४७ सैंतालीस +४८ अड़तालीस +४९ उनचास +५० पचास +५१ इक्यावन +५२ बावन +५३ तिरेपन +५४ चौवन +५५ पचपन +५६ छप्पन +५७ सत्तावन +५८ अट्ठावन +५९ उनसठ +६० साठ +६१ इकसठ +६२ बासठ +६३ तिरेसठ +६४ चौंसठ +६५ पैंसठ +६६ छियासठ +६७ सड़सठ +६८ अड़सठ +६९ उनहत्तर +७० सत्तर +७१ इकहत्तर +७२ बहत्तर +७३ तिहत्तर +७४ चौहत्तर +७५ पचहत्तर +७६ छिहत्तर +७७ सतहत्तर +७८ अठहत्तर +७९ उनासी +८० अस्सी +८१ इक्यासी +८२ बयासी +८३ तिरासी +८४ चौरासी +८५ पचासी +८६ छियासी +८७ सत्तासी +८८ अट्ठासी +८९ नवासी +९० नब्बे +९१ इक्यानबे +९२ बानबे +९३ तिरानबे +९४ चौरानबे +९५ पंचानबे +९६ छियानबे +९७ सत्तानबे +९८ अट्ठानबे +९९ निन्यानबे diff --git a/nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties_en.tsv b/nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties_en.tsv new file mode 100644 index 000000000..59071c8e0 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/numbers/teens_and_ties_en.tsv @@ -0,0 +1,90 @@ +10 दस +11 ग्यारह +12 बारह +13 तेरह +14 चौदह +15 पंद्रह +16 सोलह +17 सत्रह +18 अठारह +19 उन्नीस +20 बीस +21 इक्कीस +22 बाईस +23 तेईस +24 चौबीस +25 पच्चीस +26 छब्बीस +27 सत्ताईस +28 अट्ठाईस +29 उनतीस +30 तीस +31 इकतीस +32 बत्तीस +33 तैंतीस +34 चौंतीस +35 पैंतीस +36 छत्तीस +37 सैंतीस +38 अड़तीस +39 उनतालीस +40 चालीस +41 इकतालीस +42 बयालीस +43 तैंतालीस +44 चौवालीस +45 पैंतालीस +46 छियालीस +47 सैंतालीस +48 अड़तालीस +49 उनचास +50 पचास +51 इक्यावन +52 बावन +53 तिरेपन +54 चौवन +55 पचपन +56 छप्पन +57 सत्तावन +58 अट्ठावन +59 उनसठ +60 साठ +61 इकसठ +62 बासठ +63 तिरेसठ +64 चौंसठ +65 पैंसठ +66 छियासठ +67 सड़सठ +68 अड़सठ +69 उनहत्तर +70 सत्तर +71 इकहत्तर +72 बहत्तर +73 तिहत्तर +74 चौहत्तर +75 पचहत्तर +76 छिहत्तर +77 सतहत्तर +78 अठहत्तर +79 उनासी +80 अस्सी +81 इक्यासी +82 बयासी +83 तिरासी +84 चौरासी +85 पचासी +86 छियासी +87 सत्तासी +88 अट्ठासी +89 नवासी +90 नब्बे +91 इक्यानबे +92 बानबे +93 तिरानबे +94 चौरानबे +95 पंचानबे +96 छियानबे +97 सत्तानबे +98 अट्ठानबे +99 निन्यानबे diff --git a/nemo_text_processing/text_normalization/hi/data/numbers/thousands.tsv b/nemo_text_processing/text_normalization/hi/data/numbers/thousands.tsv new file mode 100644 index 000000000..ab9889218 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/numbers/thousands.tsv @@ -0,0 +1,8 @@ +१००० हज़ार +१००००० लाख +१००००००० करोड़ +१००००००००० अरब +१००००००००००० खरब +१००००००००००००० नील +१००००००००००००००० पद्म +१००००००००००००००००० शंख diff --git a/nemo_text_processing/text_normalization/hi/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/hi/data/numbers/zero.tsv new file mode 100644 index 000000000..0e0c50b9b --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/numbers/zero.tsv @@ -0,0 +1,2 @@ +० शून्य +0 शून्य \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/hi/data/ordinal/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/ordinal/en_to_hi_digit.tsv b/nemo_text_processing/text_normalization/hi/data/ordinal/en_to_hi_digit.tsv new file mode 100644 index 000000000..a89e99b3c --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/ordinal/en_to_hi_digit.tsv @@ -0,0 +1,10 @@ +0 ० +1 १ +2 २ +3 ३ +4 ४ +5 ५ +6 ६ +7 ७ +8 ८ +9 ९ diff --git a/nemo_text_processing/text_normalization/hi/data/ordinal/exceptions.tsv b/nemo_text_processing/text_normalization/hi/data/ordinal/exceptions.tsv new file mode 100644 index 000000000..1f14d3161 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/ordinal/exceptions.tsv @@ -0,0 +1,50 @@ +१ला पहला +१ली पहली +२रा दूसरा +२री दूसरी +३रा तीसरा +३री तीसरी +४था चौथा +४थी चौथी +६ठा छठा +६ठी छठी +1ला पहला +1ली पहली +2रा दूसरा +2री दूसरी +3रा तीसरा +3री तीसरी +4था चौथा +4थी चौथी +6ठा छठा +6ठी छठी +१st फ़र्स्ट +२nd सेकंड +३rd थर्ड +४th फ़ोर्थ +५th फ़िफ्थ +६th सिक्स्थ +७th सेवंथ +८th एटथ +९th नाइंथ +१०th टेंथ +११th इलेवंथ +१२th ट्वेल्फ्थ +१३th थर्टींथ +१४th फोर्टींथ +१५th फिफ्टींथ +1st फ़र्स्ट +2nd सेकंड +3rd थर्ड +4th फ़ोर्थ +5th फ़िफ्थ +6th सिक्स्थ +7th सेवंथ +8th एटथ +9th नाइंथ +10th टेंथ +11th इलेवंथ +12th ट्वेल्फ्थ +13th थर्टींथ +14th फोर्टींथ +15th फिफ्टींथ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/ordinal/suffixes.tsv b/nemo_text_processing/text_normalization/hi/data/ordinal/suffixes.tsv new file mode 100644 index 000000000..922e9d6b8 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/ordinal/suffixes.tsv @@ -0,0 +1,3 @@ +वां +वीं +वें diff --git a/nemo_text_processing/text_normalization/hi/data/ordinal/suffixes_map.tsv b/nemo_text_processing/text_normalization/hi/data/ordinal/suffixes_map.tsv new file mode 100644 index 000000000..2abb5c492 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/ordinal/suffixes_map.tsv @@ -0,0 +1 @@ +वे वें \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/roman/__init__.py b/nemo_text_processing/text_normalization/hi/data/roman/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_ordinal_exceptions.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_ordinal_exceptions.tsv new file mode 100644 index 000000000..b298e13e6 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/roman_ordinal_exceptions.tsv @@ -0,0 +1,10 @@ +Iला पहला +Iली पहली +IIरा दूसरा +IIरी दूसरी +IIIरा तीसरा +IIIरी तीसरी +IVथा चौथा +IVथी चौथी +VIठा छठा +VIठी छठी \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv new file mode 100644 index 000000000..69b760196 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv @@ -0,0 +1,100 @@ +I एक +II दो +III तीन +IV चार +V पाँच +VI छह +VII सात +VIII आठ +IX नौ +X दस +XI ग्यारह +XII बारह +XIII तेरह +XIV चौदह +XV पंद्रह +XVI सोलह +XVII सत्रह +XVIII अठारह +XIX उन्नीस +XX बीस +XXI इक्कीस +XXII बाईस +XXIII तेईस +XXIV चौबीस +XXV पच्चीस +XXVI छब्बीस +XXVII सत्ताईस +XXVIII अट्ठाईस +XXIX उनतीस +XXX तीस +XXXI इकतीस +XXXII बत्तीस +XXXIII तैंतीस +XXXIV चौंतीस +XXXV पैंतीस +XXXVI छत्तीस +XXXVII सैंतीस +XXXVIII अड़तीस +XXXIX उनचालीस +XL चालीस +XLI इकतालीस +XLII बयालीस +XLIII तैंतालीस +XLIV चौंतालीस +XLV पैंतालीस +XLVI छियालीस +XLVII सैंतालीस +XLVIII अड़तालीस +XLIX उनचास +L पचास +LI इक्यावन +LII बावन +LIII तिरपन +LIV चौवन +LV पचपन +LVI छप्पन +LVII सत्तावन +LVIII अट्ठावन +LIX उनसठ +LX साठ +LXI इकसठ +LXII बासठ +LXIII तिरसठ +LXIV चौंसठ +LXV पैंसठ +LXVI छियासठ +LXVII सड़सठ +LXVIII अड़सठ +LXIX उनहत्तर +LXX सत्तर +LXXI इकहत्तर +LXXII बहत्तर +LXXIII तिहत्तर +LXXIV चौहत्तर +LXXV पचहत्तर +LXXVI छिहत्तर +LXXVII सतहत्तर +LXXVIII अठहत्तर +LXXIX उनासी +LXXX अस्सी +LXXXI इक्यासी +LXXXII बयासी +LXXXIII तिरासी +LXXXIV चौरासी +LXXXV पचासी +LXXXVI छियासी +LXXXVII सत्तासी +LXXXVIII अट्ठासी +LXXXIX नवासी +XC नब्बे +XCI इक्यानवे +XCII बानवे +XCIII तिरानवे +XCIV चौरानवे +XCV पचानवे +XCVI छियानवे +XCVII सत्तानवे +XCVIII अट्ठानवे +XCIX निन्यानवे +C एक सौ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/serial/__init__.py b/nemo_text_processing/text_normalization/hi/data/serial/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/serial/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/serial/chars.tsv b/nemo_text_processing/text_normalization/hi/data/serial/chars.tsv new file mode 100644 index 000000000..d7c9c39e8 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/serial/chars.tsv @@ -0,0 +1,68 @@ +अ +आ +इ +ई +उ +ऊ +ऋ +ए +ऐ +ओ +औ +ऑ +ा +ि +ी +ु +ू +ृ +े +ै +ो +ौ +ॉ +ं +ः +ँ +क +ख +ग +घ +ङ +च +छ +ज +झ +ञ +ट +ठ +ड +ढ +ण +त +थ +द +ध +न +प +फ +ब +भ +म +य +र +ल +व +श +ष +स +ह +क़ +ख़ +ग़ +ज़ +ड़ +ढ़ +फ़ +य़ +् \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/serial/power_special.tsv b/nemo_text_processing/text_normalization/hi/data/serial/power_special.tsv new file mode 100644 index 000000000..64583f947 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/serial/power_special.tsv @@ -0,0 +1,4 @@ +^2 स्क्वेर्ड +^२ स्क्वेर्ड +^3 क्यूब +^३ क्यूब \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/serial/special_symbols.tsv b/nemo_text_processing/text_normalization/hi/data/serial/special_symbols.tsv new file mode 100644 index 000000000..c96a15bd6 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/serial/special_symbols.tsv @@ -0,0 +1,4 @@ +# हैशटैग +% प्रतिशत +& एंड +@ एट \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/telephone/__init__.py b/nemo_text_processing/text_normalization/hi/data/telephone/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/telephone/credit_context.tsv b/nemo_text_processing/text_normalization/hi/data/telephone/credit_context.tsv new file mode 100644 index 000000000..46b485af6 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/telephone/credit_context.tsv @@ -0,0 +1,3 @@ +नंबर +कार्ड +क्रेडिट \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/telephone/landline_context.tsv b/nemo_text_processing/text_normalization/hi/data/telephone/landline_context.tsv new file mode 100644 index 000000000..17a123bee --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/telephone/landline_context.tsv @@ -0,0 +1,5 @@ +नंबर +मोबाइल +फोन +लैंडलाइन +कॉल \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/telephone/mobile_context.tsv b/nemo_text_processing/text_normalization/hi/data/telephone/mobile_context.tsv new file mode 100644 index 000000000..f2fa6e52f --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/telephone/mobile_context.tsv @@ -0,0 +1,4 @@ +नंबर +मोबाइल +फोन +कॉल \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/telephone/number.tsv b/nemo_text_processing/text_normalization/hi/data/telephone/number.tsv new file mode 100644 index 000000000..e8c04b723 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/telephone/number.tsv @@ -0,0 +1,10 @@ +0 शून्य +1 एक +2 दो +3 तीन +4 चार +5 पाँच +6 छह +7 सात +8 आठ +9 नौ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/telephone/pincode_context.tsv b/nemo_text_processing/text_normalization/hi/data/telephone/pincode_context.tsv new file mode 100644 index 000000000..322c7248e --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/telephone/pincode_context.tsv @@ -0,0 +1,4 @@ +नंबर +पिन +कोड +पिनकोड \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/time/__init__.py b/nemo_text_processing/text_normalization/hi/data/time/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/time/hours.tsv b/nemo_text_processing/text_normalization/hi/data/time/hours.tsv new file mode 100644 index 000000000..7d6e552f3 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/time/hours.tsv @@ -0,0 +1,50 @@ +० शून्य +१ एक +२ दो +३ तीन +४ चार +५ पाँच +६ छह +७ सात +८ आठ +९ नौ +१० दस +११ ग्यारह +१२ बारह +१३ तेरह +१४ चौदह +१५ पंद्रह +१६ सोलह +१७ सत्रह +१८ अठारह +१९ उन्नीस +२० बीस +२१ इक्कीस +२२ बाईस +२३ तेईस +२४ चौबीस +0 शून्य +1 एक +2 दो +3 तीन +4 चार +5 पाँच +6 छह +7 सात +8 आठ +9 नौ +10 दस +11 ग्यारह +12 बारह +13 तेरह +14 चौदह +15 पंद्रह +16 सोलह +17 सत्रह +18 अठारह +19 उन्नीस +20 बीस +21 इक्कीस +22 बाईस +23 तेईस +24 चौबीस \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv b/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv new file mode 100644 index 000000000..895ab481a --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/time/minutes.tsv @@ -0,0 +1,120 @@ +०१ एक +०२ दो +०३ तीन +०४ चार +०५ पाँच +०६ छह +०७ सात +०८ आठ +०९ नौ +१० दस +११ ग्यारह +१२ बारह +१३ तेरह +१४ चौदह +१५ पंद्रह +१६ सोलह +१७ सत्रह +१८ अठारह +१९ उन्नीस +२० बीस +२१ इक्कीस +२२ बाईस +२३ तेईस +२४ चौबीस +२५ पच्चीस +२६ छब्बीस +२७ सत्ताईस +२८ अट्ठाईस +२९ उनतीस +३० तीस +३१ इकतीस +३२ बत्तीस +३३ तैंतीस +३४ चौंतीस +३५ पैंतीस +३६ छत्तीस +३७ सैंतीस +३८ अड़तीस +३९ उनतालीस +४० चालीस +४१ इकतालीस +४२ बयालीस +४३ तैंतालीस +४४ चौवालीस +४५ पैंतालीस +४६ छियालीस +४७ सैंतालीस +४८ अड़तालीस +४९ उनचास +५० पचास +५१ इक्यावन +५२ बावन +५३ तिरेपन +५४ चौवन +५५ पचपन +५६ छप्पन +५७ सत्तावन +५८ अट्ठावन +५९ उनसठ +६० साठ +01 एक +02 दो +03 तीन +04 चार +05 पाँच +06 छह +07 सात +08 आठ +09 नौ +10 दस +11 ग्यारह +12 बारह +13 तेरह +14 चौदह +15 पंद्रह +16 सोलह +17 सत्रह +18 अठारह +19 उन्नीस +20 बीस +21 इक्कीस +22 बाईस +23 तेईस +24 चौबीस +25 पच्चीस +26 छब्बीस +27 सत्ताईस +28 अट्ठाईस +29 उनतीस +30 तीस +31 इकतीस +32 बत्तीस +33 तैंतीस +34 चौंतीस +35 पैंतीस +36 छत्तीस +37 सैंतीस +38 अड़तीस +39 उनतालीस +40 चालीस +41 इकतालीस +42 बयालीस +43 तैंतालीस +44 चौवालीस +45 पैंतालीस +46 छियालीस +47 सैंतालीस +48 अड़तालीस +49 उनचास +50 पचास +51 इक्यावन +52 बावन +53 तिरेपन +54 चौवन +55 पचपन +56 छप्पन +57 सत्तावन +58 अट्ठावन +59 उनसठ +60 साठ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv b/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv new file mode 100644 index 000000000..895ab481a --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/time/seconds.tsv @@ -0,0 +1,120 @@ +०१ एक +०२ दो +०३ तीन +०४ चार +०५ पाँच +०६ छह +०७ सात +०८ आठ +०९ नौ +१० दस +११ ग्यारह +१२ बारह +१३ तेरह +१४ चौदह +१५ पंद्रह +१६ सोलह +१७ सत्रह +१८ अठारह +१९ उन्नीस +२० बीस +२१ इक्कीस +२२ बाईस +२३ तेईस +२४ चौबीस +२५ पच्चीस +२६ छब्बीस +२७ सत्ताईस +२८ अट्ठाईस +२९ उनतीस +३० तीस +३१ इकतीस +३२ बत्तीस +३३ तैंतीस +३४ चौंतीस +३५ पैंतीस +३६ छत्तीस +३७ सैंतीस +३८ अड़तीस +३९ उनतालीस +४० चालीस +४१ इकतालीस +४२ बयालीस +४३ तैंतालीस +४४ चौवालीस +४५ पैंतालीस +४६ छियालीस +४७ सैंतालीस +४८ अड़तालीस +४९ उनचास +५० पचास +५१ इक्यावन +५२ बावन +५३ तिरेपन +५४ चौवन +५५ पचपन +५६ छप्पन +५७ सत्तावन +५८ अट्ठावन +५९ उनसठ +६० साठ +01 एक +02 दो +03 तीन +04 चार +05 पाँच +06 छह +07 सात +08 आठ +09 नौ +10 दस +11 ग्यारह +12 बारह +13 तेरह +14 चौदह +15 पंद्रह +16 सोलह +17 सत्रह +18 अठारह +19 उन्नीस +20 बीस +21 इक्कीस +22 बाईस +23 तेईस +24 चौबीस +25 पच्चीस +26 छब्बीस +27 सत्ताईस +28 अट्ठाईस +29 उनतीस +30 तीस +31 इकतीस +32 बत्तीस +33 तैंतीस +34 चौंतीस +35 पैंतीस +36 छत्तीस +37 सैंतीस +38 अड़तीस +39 उनतालीस +40 चालीस +41 इकतालीस +42 बयालीस +43 तैंतालीस +44 चौवालीस +45 पैंतालीस +46 छियालीस +47 सैंतालीस +48 अड़तालीस +49 उनचास +50 पचास +51 इक्यावन +52 बावन +53 तिरेपन +54 चौवन +55 पचपन +56 छप्पन +57 सत्तावन +58 अट्ठावन +59 उनसठ +60 साठ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/whitelist/__init__.py b/nemo_text_processing/text_normalization/hi/data/whitelist/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/data/whitelist/abbreviations.tsv b/nemo_text_processing/text_normalization/hi/data/whitelist/abbreviations.tsv new file mode 100644 index 000000000..fb39da2a4 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/whitelist/abbreviations.tsv @@ -0,0 +1,7 @@ +डॉ. डॉक्टर +प्रो. प्रोफेसर +इं. इंजीनियर +ले. लेफ्टिनेंट +वै. वैज्ञानिक +कु. कुमारी +मा. मास्टर \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv b/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv new file mode 100644 index 000000000..3477871e4 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv @@ -0,0 +1,100 @@ +० एक +१ दो +२ तीन +३ चार +४ पाँच +५ छह +६ सात +७ आठ +८ नौ +९ दस +१० ग्यारह +११ बारह +१२ तेरह +१३ चौदह +१४ पंद्रह +१५ सोलह +१६ सत्रह +१७ अठारह +१८ उन्नीस +१९ बीस +२० इक्कीस +२१ बाईस +२२ तेईस +२३ चौबीस +२४ पच्चीस +२५ छब्बीस +२६ सत्ताईस +२७ अट्ठाईस +२८ उनतीस +२९ तीस +३० इकतीस +३१ बत्तीस +३२ तैंतीस +३३ चौंतीस +३४ पैंतीस +३५ छत्तीस +३६ सैंतीस +३७ अड़तीस +३८ उनतालीस +३९ चालीस +४० इकतालीस +४१ बयालीस +४२ तैंतालीस +४३ चौवालीस +४४ पैंतालीस +४५ छियालीस +४६ सैंतालीस +४७ अड़तालीस +४८ उनचास +४९ पचास +५० इक्यावन +५१ बावन +५२ तिरेपन +५३ चौवन +५४ पचपन +५५ छप्पन +५६ सत्तावन +५७ अट्ठावन +५८ उनसठ +५९ साठ +६० इकसठ +६१ बासठ +६२ तिरेसठ +६३ चौंसठ +६४ पैंसठ +६५ छियासठ +६६ सड़सठ +६७ अड़सठ +६८ उनहत्तर +६९ सत्तर +७० इकहत्तर +७१ बहत्तर +७२ तिहत्तर +७३ चौहत्तर +७४ पचहत्तर +७५ छिहत्तर +७६ सतहत्तर +७७ अठहत्तर +७८ उनासी +७९ अस्सी +८० इक्यासी +८१ बयासी +८२ तिरासी +८३ चौरासी +८४ पचासी +८५ छियासी +८६ सत्तासी +८७ अट्ठासी +८८ नवासी +८९ नब्बे +९० इक्यानबे +९१ बानबे +९२ तिरानबे +९३ चौरानबे +९४ पंचानबे +९५ छियानबे +९६ सत्तानबे +९७ अट्ठानबे +९८ निन्यानबे +९९ एक सौ diff --git a/nemo_text_processing/text_normalization/hi/graph_utils.py b/nemo_text_processing/text_normalization/hi/graph_utils.py new file mode 100644 index 000000000..e8543d9ad --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/graph_utils.py @@ -0,0 +1,225 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +NEMO_CHAR = utf8.VALID_UTF8_CHAR +NEMO_DIGIT = byte.DIGIT + +NEMO_HI_DIGIT = pynini.union("०", "१", "२", "३", "४", "५", "६", "७", "८", "९").optimize() +NEMO_HI_NON_ZERO = pynini.union("१", "२", "३", "४", "५", "६", "७", "८", "९").optimize() +NEMO_HI_ZERO = "०" +# Combined Hindi and Arabic digits for graphs that need to accept both +NEMO_ALL_DIGIT = pynini.union(NEMO_HI_DIGIT, NEMO_DIGIT).optimize() +NEMO_ALL_ZERO = pynini.union("०", "0").optimize() +NEMO_ALL_NON_ZERO = pynini.union(NEMO_HI_NON_ZERO, "1", "2", "3", "4", "5", "6", "7", "8", "9").optimize() + +HI_DEDH = "डेढ़" # 1.5 +HI_DHAI = "ढाई" # 2.5 +HI_SAVVA = "सवा" # quarter more (1.25) +HI_SADHE = "साढ़े" # half more (X.5) +HI_PAUNE = "पौने" # quarter less (0.75) + +# Hindi decimal representations +HI_POINT_FIVE = ".५" # .5 +HI_ONE_POINT_FIVE = "१.५" # 1.5 +HI_TWO_POINT_FIVE = "२.५" # 2.5 +HI_DECIMAL_25 = ".२५" # .25 +HI_DECIMAL_75 = ".७५" # .75 + +# Arabic/English decimal representations +EN_POINT_FIVE = ".5" +EN_ONE_POINT_FIVE = "1.5" +EN_TWO_POINT_FIVE = "2.5" +EN_DECIMAL_25 = ".25" +EN_DECIMAL_75 = ".75" + +# Combined Hindi and English decimal patterns +POINT_FIVE = pynini.union(HI_POINT_FIVE, EN_POINT_FIVE).optimize() +ONE_POINT_FIVE = pynini.union(HI_ONE_POINT_FIVE, EN_ONE_POINT_FIVE).optimize() +TWO_POINT_FIVE = pynini.union(HI_TWO_POINT_FIVE, EN_TWO_POINT_FIVE).optimize() +DECIMAL_25 = pynini.union(HI_DECIMAL_25, EN_DECIMAL_25).optimize() +DECIMAL_75 = pynini.union(HI_DECIMAL_75, EN_DECIMAL_75).optimize() + +# Symbol constants +HI_BY = "बाई" +LOWERCASE_X = "x" +UPPERCASE_X = "X" +ASTERISK = "*" +HYPHEN = "-" +SLASH = "/" +COMMA = "," +PERIOD = "." +HI_PERIOD = "।" + +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union(" ऋणात्मक ", " ऋणात्मक ").optimize() + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/hi/taggers/__init__.py b/nemo_text_processing/text_normalization/hi/taggers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/taggers/cardinal.py b/nemo_text_processing/text_normalization/hi/taggers/cardinal.py new file mode 100644 index 000000000..dd4611010 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/cardinal.py @@ -0,0 +1,395 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + NEMO_ALL_DIGIT, + NEMO_ALL_ZERO, + NEMO_DIGIT, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + -२३ -> cardinal { negative: "true" integer: "तेइस" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + # Load both Hindi (Devanagari) and English (Arabic) number mappings + teens_ties_hi = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")) + teens_ties_en = pynini.string_file(get_abs_path("data/numbers/teens_and_ties_en.tsv")) + teens_ties = pynini.union(teens_ties_hi, teens_ties_en) + teens_and_ties = pynutil.add_weight(teens_ties, -0.1) + + self.digit = digit + self.zero = zero + self.teens_and_ties = teens_and_ties + + # Single digit graph for digit-by-digit reading + # e.g., "०७३" -> "शून्य सात तीन" + single_digit_graph = digit | zero + self.single_digits_graph = single_digit_graph + pynini.closure(insert_space + single_digit_graph) + + def create_graph_suffix(digit_graph, suffix, zeros_counts): + zero = pynutil.add_weight(pynutil.delete(NEMO_ALL_ZERO), -0.1) + if zeros_counts == 0: + return digit_graph + suffix + + return digit_graph + (zero**zeros_counts) + suffix + + def create_larger_number_graph(digit_graph, suffix, zeros_counts, sub_graph): + insert_space = pynutil.insert(" ") + zero = pynutil.add_weight(pynutil.delete(NEMO_ALL_ZERO), -0.1) + if zeros_counts == 0: + return digit_graph + suffix + insert_space + sub_graph + + return digit_graph + suffix + (zero**zeros_counts) + insert_space + sub_graph + + # Hundred graph + suffix_hundreds = pynutil.insert(" सौ") + graph_hundreds = create_graph_suffix(digit, suffix_hundreds, 2) + graph_hundreds |= create_larger_number_graph(digit, suffix_hundreds, 1, digit) + graph_hundreds |= create_larger_number_graph(digit, suffix_hundreds, 0, teens_ties) + graph_hundreds.optimize() + self.graph_hundreds = graph_hundreds + + # Transducer for eleven hundred -> 1100 or twenty one hundred eleven -> 2111 + graph_hundreds_as_thousand = create_graph_suffix(teens_and_ties, suffix_hundreds, 2) + graph_hundreds_as_thousand |= create_larger_number_graph(teens_and_ties, suffix_hundreds, 1, digit) + graph_hundreds_as_thousand |= create_larger_number_graph(teens_and_ties, suffix_hundreds, 0, teens_ties) + self.graph_hundreds_as_thousand = graph_hundreds_as_thousand + + # Thousands and Ten thousands graph + suffix_thousands = pynutil.insert(" हज़ार") + graph_thousands = create_graph_suffix(digit, suffix_thousands, 3) + graph_thousands |= create_larger_number_graph(digit, suffix_thousands, 2, digit) + graph_thousands |= create_larger_number_graph(digit, suffix_thousands, 1, teens_ties) + graph_thousands |= create_larger_number_graph(digit, suffix_thousands, 0, graph_hundreds) + graph_thousands.optimize() + self.graph_thousands = graph_thousands + + graph_ten_thousands = create_graph_suffix(teens_and_ties, suffix_thousands, 3) + graph_ten_thousands |= create_larger_number_graph(teens_and_ties, suffix_thousands, 2, digit) + graph_ten_thousands |= create_larger_number_graph(teens_and_ties, suffix_thousands, 1, teens_ties) + graph_ten_thousands |= create_larger_number_graph(teens_and_ties, suffix_thousands, 0, graph_hundreds) + graph_ten_thousands.optimize() + self.graph_ten_thousands = graph_ten_thousands + + # Lakhs graph and ten lakhs graph + suffix_lakhs = pynutil.insert(" लाख") + graph_lakhs = create_graph_suffix(digit, suffix_lakhs, 5) + graph_lakhs |= create_larger_number_graph(digit, suffix_lakhs, 4, digit) + graph_lakhs |= create_larger_number_graph(digit, suffix_lakhs, 3, teens_ties) + graph_lakhs |= create_larger_number_graph(digit, suffix_lakhs, 2, graph_hundreds) + graph_lakhs |= create_larger_number_graph(digit, suffix_lakhs, 1, graph_thousands) + graph_lakhs |= create_larger_number_graph(digit, suffix_lakhs, 0, graph_ten_thousands) + graph_lakhs.optimize() + self.graph_lakhs = graph_lakhs + + graph_ten_lakhs = create_graph_suffix(teens_and_ties, suffix_lakhs, 5) + graph_ten_lakhs |= create_larger_number_graph(teens_and_ties, suffix_lakhs, 4, digit) + graph_ten_lakhs |= create_larger_number_graph(teens_and_ties, suffix_lakhs, 3, teens_ties) + graph_ten_lakhs |= create_larger_number_graph(teens_and_ties, suffix_lakhs, 2, graph_hundreds) + graph_ten_lakhs |= create_larger_number_graph(teens_and_ties, suffix_lakhs, 1, graph_thousands) + graph_ten_lakhs |= create_larger_number_graph(teens_and_ties, suffix_lakhs, 0, graph_ten_thousands) + graph_ten_lakhs.optimize() + self.graph_ten_lakhs = graph_ten_lakhs + + # Crores graph ten crores graph + suffix_crores = pynutil.insert(" करोड़") + graph_crores = create_graph_suffix(digit, suffix_crores, 7) + graph_crores |= create_larger_number_graph(digit, suffix_crores, 6, digit) + graph_crores |= create_larger_number_graph(digit, suffix_crores, 5, teens_ties) + graph_crores |= create_larger_number_graph(digit, suffix_crores, 4, graph_hundreds) + graph_crores |= create_larger_number_graph(digit, suffix_crores, 3, graph_thousands) + graph_crores |= create_larger_number_graph(digit, suffix_crores, 2, graph_ten_thousands) + graph_crores |= create_larger_number_graph(digit, suffix_crores, 1, graph_lakhs) + graph_crores |= create_larger_number_graph(digit, suffix_crores, 0, graph_ten_lakhs) + graph_crores.optimize() + + graph_ten_crores = create_graph_suffix(teens_and_ties, suffix_crores, 7) + graph_ten_crores |= create_larger_number_graph(teens_and_ties, suffix_crores, 6, digit) + graph_ten_crores |= create_larger_number_graph(teens_and_ties, suffix_crores, 5, teens_ties) + graph_ten_crores |= create_larger_number_graph(teens_and_ties, suffix_crores, 4, graph_hundreds) + graph_ten_crores |= create_larger_number_graph(teens_and_ties, suffix_crores, 3, graph_thousands) + graph_ten_crores |= create_larger_number_graph(teens_and_ties, suffix_crores, 2, graph_ten_thousands) + graph_ten_crores |= create_larger_number_graph(teens_and_ties, suffix_crores, 1, graph_lakhs) + graph_ten_crores |= create_larger_number_graph(teens_and_ties, suffix_crores, 0, graph_ten_lakhs) + graph_ten_crores.optimize() + + # Arabs graph and ten arabs graph + suffix_arabs = pynutil.insert(" अरब") + graph_arabs = create_graph_suffix(digit, suffix_arabs, 9) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 8, digit) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 7, teens_ties) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 6, graph_hundreds) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 5, graph_thousands) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 4, graph_ten_thousands) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 3, graph_lakhs) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 2, graph_ten_lakhs) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 1, graph_crores) + graph_arabs |= create_larger_number_graph(digit, suffix_arabs, 0, graph_ten_crores) + graph_arabs.optimize() + + graph_ten_arabs = create_graph_suffix(teens_and_ties, suffix_arabs, 9) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 8, digit) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 7, teens_ties) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 6, graph_hundreds) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 5, graph_thousands) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 4, graph_ten_thousands) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 3, graph_lakhs) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 2, graph_ten_lakhs) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 1, graph_crores) + graph_ten_arabs |= create_larger_number_graph(teens_and_ties, suffix_arabs, 0, graph_ten_crores) + graph_ten_arabs.optimize() + + # Kharabs graph and ten kharabs graph + suffix_kharabs = pynutil.insert(" खरब") + graph_kharabs = create_graph_suffix(digit, suffix_kharabs, 11) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 10, digit) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 9, teens_ties) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 8, graph_hundreds) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 7, graph_thousands) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 6, graph_ten_thousands) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 5, graph_lakhs) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 4, graph_ten_lakhs) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 3, graph_crores) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 2, graph_ten_crores) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 1, graph_arabs) + graph_kharabs |= create_larger_number_graph(digit, suffix_kharabs, 0, graph_ten_arabs) + graph_kharabs.optimize() + + graph_ten_kharabs = create_graph_suffix(teens_and_ties, suffix_kharabs, 11) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 10, digit) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 9, teens_ties) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 8, graph_hundreds) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 7, graph_thousands) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 6, graph_ten_thousands) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 5, graph_lakhs) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 4, graph_ten_lakhs) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 3, graph_crores) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 2, graph_ten_crores) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 1, graph_arabs) + graph_ten_kharabs |= create_larger_number_graph(teens_and_ties, suffix_kharabs, 0, graph_ten_arabs) + graph_ten_kharabs.optimize() + + # Nils graph and ten nils graph + suffix_nils = pynutil.insert(" नील") + graph_nils = create_graph_suffix(digit, suffix_nils, 13) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 12, digit) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 11, teens_ties) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 10, graph_hundreds) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 9, graph_thousands) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 8, graph_ten_thousands) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 7, graph_lakhs) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 6, graph_ten_lakhs) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 5, graph_crores) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 4, graph_ten_crores) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 3, graph_arabs) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 2, graph_ten_arabs) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 1, graph_kharabs) + graph_nils |= create_larger_number_graph(digit, suffix_nils, 0, graph_ten_kharabs) + graph_nils.optimize() + + graph_ten_nils = create_graph_suffix(teens_and_ties, suffix_nils, 13) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 12, digit) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 11, teens_ties) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 10, graph_hundreds) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 9, graph_thousands) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 8, graph_ten_thousands) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 7, graph_lakhs) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 6, graph_ten_lakhs) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 5, graph_crores) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 4, graph_ten_crores) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 3, graph_arabs) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 2, graph_ten_arabs) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 1, graph_kharabs) + graph_ten_nils |= create_larger_number_graph(teens_and_ties, suffix_nils, 0, graph_ten_kharabs) + graph_ten_nils.optimize() + + # Padmas graph and ten padmas graph + suffix_padmas = pynutil.insert(" पद्म") + graph_padmas = create_graph_suffix(digit, suffix_padmas, 15) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 14, digit) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 13, teens_ties) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 12, graph_hundreds) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 11, graph_thousands) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 10, graph_ten_thousands) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 9, graph_lakhs) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 8, graph_ten_lakhs) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 7, graph_crores) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 6, graph_ten_crores) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 5, graph_arabs) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 4, graph_ten_arabs) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 3, graph_kharabs) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 2, graph_ten_kharabs) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 1, graph_nils) + graph_padmas |= create_larger_number_graph(digit, suffix_padmas, 0, graph_ten_nils) + graph_padmas.optimize() + + graph_ten_padmas = create_graph_suffix(teens_and_ties, suffix_padmas, 15) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 14, digit) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 13, teens_ties) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 12, graph_hundreds) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 11, graph_thousands) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 10, graph_ten_thousands) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 9, graph_lakhs) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 8, graph_ten_lakhs) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 7, graph_crores) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 6, graph_ten_crores) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 5, graph_arabs) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 4, graph_ten_arabs) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 3, graph_kharabs) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 2, graph_ten_kharabs) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 1, graph_nils) + graph_ten_padmas |= create_larger_number_graph(teens_and_ties, suffix_padmas, 0, graph_ten_nils) + graph_ten_padmas.optimize() + + # Shankhs graph and ten shankhs graph + suffix_shankhs = pynutil.insert(" शंख") + graph_shankhs = create_graph_suffix(digit, suffix_shankhs, 17) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 16, digit) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 15, teens_ties) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 14, graph_hundreds) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 13, graph_thousands) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 12, graph_ten_thousands) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 11, graph_lakhs) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 10, graph_ten_lakhs) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 9, graph_crores) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 8, graph_ten_crores) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 7, graph_arabs) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 6, graph_ten_arabs) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 5, graph_kharabs) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 4, graph_ten_kharabs) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 3, graph_nils) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 2, graph_ten_nils) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 1, graph_padmas) + graph_shankhs |= create_larger_number_graph(digit, suffix_shankhs, 0, graph_ten_padmas) + graph_shankhs.optimize() + + graph_ten_shankhs = create_graph_suffix(teens_and_ties, suffix_shankhs, 17) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 16, digit) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 15, teens_ties) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 14, graph_hundreds) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 13, graph_thousands) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 12, graph_ten_thousands) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 11, graph_lakhs) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 10, graph_ten_lakhs) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 9, graph_crores) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 8, graph_ten_crores) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 7, graph_arabs) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 6, graph_ten_arabs) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 5, graph_kharabs) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 4, graph_ten_kharabs) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 3, graph_nils) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 2, graph_ten_nils) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 1, graph_padmas) + graph_ten_shankhs |= create_larger_number_graph(teens_and_ties, suffix_shankhs, 0, graph_ten_padmas) + graph_ten_shankhs.optimize() + + # Only match exactly 2 digits to avoid interfering with telephone numbers, decimals, etc. + # e.g., "०५" -> "शून्य पाँच" + single_digit = digit | zero + graph_leading_zero = zero + insert_space + single_digit + graph_leading_zero = pynutil.add_weight(graph_leading_zero, 0.5) + + graph_without_leading_zeros = ( + digit + | zero + | teens_and_ties + | graph_hundreds + | graph_thousands + | graph_ten_thousands + | graph_lakhs + | graph_ten_lakhs + | graph_crores + | graph_ten_crores + | graph_arabs + | graph_ten_arabs + | graph_kharabs + | graph_ten_kharabs + | graph_nils + | graph_ten_nils + | graph_padmas + | graph_ten_padmas + | graph_shankhs + | graph_ten_shankhs + ) + self.graph_without_leading_zeros = graph_without_leading_zeros.optimize() + + # Handle numbers with leading zeros by reading digit-by-digit + # e.g., English/arabic "073" -> "शून्य सात तीन", Hindi/devnagri "००५" -> "शून्य शून्य पाँच" + cardinal_with_leading_zeros = pynini.compose( + NEMO_ALL_ZERO + pynini.closure(NEMO_ALL_DIGIT), self.single_digits_graph + ) + cardinal_with_leading_zeros = pynutil.add_weight(cardinal_with_leading_zeros, 0.5) + + # Handle large numbers written with digit-group separators. + delete_separator = pynutil.delete(",") + two_digits = NEMO_ALL_DIGIT + NEMO_ALL_DIGIT + three_digits = NEMO_ALL_DIGIT + NEMO_ALL_DIGIT + NEMO_ALL_DIGIT + # Indian grouping: 1-2 leading digits, groups of 2, final group of 3. + indian_grouping = ( + pynini.closure(NEMO_ALL_DIGIT, 1, 2) + + pynini.closure(delete_separator + two_digits) + + delete_separator + + three_digits + ) + # International grouping: 1-3 leading digits, one or more groups of 3. + western_grouping = pynini.closure(NEMO_ALL_DIGIT, 1, 3) + pynini.closure(delete_separator + three_digits, 1) + strip_separators = (indian_grouping | western_grouping).optimize() + cardinal_with_separators = pynini.compose(strip_separators, graph_without_leading_zeros).optimize() + + # Full graph including leading zeros - for standalone cardinal matching + final_graph = graph_without_leading_zeros | cardinal_with_leading_zeros | cardinal_with_separators + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + + # --- Centralized logic for Address & Serial classes --- + # 1-3 digit groups read as cardinals, 4+ digits read digit-by-digit + limited_cardinal_graph = (self.digit | self.zero | self.teens_and_ties | self.graph_hundreds).optimize() + + any_digit = pynini.union( + NEMO_DIGIT, + pynini.project( + pynini.union( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")), + pynini.string_file(get_abs_path("data/numbers/zero.tsv")), + ), + "input", + ), + ).optimize() + + digitwise_4plus = pynini.compose(any_digit**4 + pynini.closure(any_digit), self.single_digits_graph).optimize() + + self.code_num_graph = (limited_cardinal_graph | digitwise_4plus).optimize() + + self.final_graph = final_graph.optimize() + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph diff --git a/nemo_text_processing/text_normalization/hi/taggers/date.py b/nemo_text_processing/text_normalization/hi/taggers/date.py new file mode 100644 index 000000000..6f43e5d6e --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/date.py @@ -0,0 +1,249 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + NEMO_ALL_DIGIT, + NEMO_ALL_NON_ZERO, + NEMO_ALL_ZERO, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + +days = pynini.string_file(get_abs_path("data/date/days.tsv")) +months = pynini.string_file(get_abs_path("data/date/months.tsv")) +year_suffix = pynini.string_file(get_abs_path("data/date/year_suffix.tsv")) +digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +teens_ties_hi = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")) +teens_ties_en = pynini.string_file(get_abs_path("data/numbers/teens_and_ties_en.tsv")) +teens_ties = pynini.union(teens_ties_hi, teens_ties_en) +teens_and_ties = pynutil.add_weight(teens_ties, -0.1) + +with open(get_abs_path("data/date/suffixes.tsv"), "r", encoding="utf-8") as f: + suffix_union = pynini.string_map([line.rstrip("\n") for line in f if line.strip()]) + +with open(get_abs_path("data/date/prefixes.tsv"), "r", encoding="utf-8") as f: + prefix_union = pynini.string_map([line.rstrip("\n") for line in f if line.strip()]) + +verbalized_hundreds = teens_ties_hi.project("output") +verbalized_unit = pynini.union(verbalized_hundreds, digit.project("output")) + +verbalized_year_sou = ( + verbalized_hundreds + pynini.accep(" सौ") + pynini.closure(pynini.accep(" ") + verbalized_unit, 0, 1) +) + +pad_latin = pynini.union(*[pynini.cross(str(i), f"0{i}") for i in range(1, 10)]) +pad_devanagari = pynini.union(*[pynini.cross(d, f"०{d}") for d in "१२३४५६७८९"]) + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying date, e.g. + "०१-०४-२०२४" -> date { day: "एक" month: "अप्रैल" year: "दो हज़ार चौबीस" } + "६ मार्च, २०१०" -> date { day: "छह" month: "मार्च" year: "दो हज़ार दस" } + "३१ मई, १९९० ई." -> date { day: "इकतीस" month: "मई" year: "उन्नीस सौ नब्बे" era: "ईसवी" } + "उन्नीस सौ बीस में" -> date { era: "उन्नीस सौ बीस में" } + "02-07-1970" -> date { day: "दो" month: "जुलाई" year: "उन्नीस सौ सत्तर" } + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="date", kind="classify") + + graph_year_thousands = pynini.compose( + (NEMO_ALL_DIGIT + NEMO_ALL_ZERO + NEMO_ALL_DIGIT + NEMO_ALL_DIGIT), cardinal.graph_thousands + ) + graph_year_hundreds_as_thousands = pynini.compose( + (NEMO_ALL_DIGIT + NEMO_ALL_NON_ZERO + NEMO_ALL_DIGIT + NEMO_ALL_DIGIT), cardinal.graph_hundreds_as_thousand + ) + + cardinal_graph = pynini.union( + digit, + teens_and_ties, + cardinal.graph_hundreds, + graph_year_thousands, + graph_year_hundreds_as_thousands, + ) + + graph_year = pynini.union(graph_year_thousands, graph_year_hundreds_as_thousands) + + graph_year_era = pynini.union( + graph_year_thousands, + graph_year_hundreds_as_thousands, + cardinal.graph_hundreds, + ) + + delete_dash = pynutil.delete("-") + delete_comma = pynutil.delete(",") + delete_space = pynutil.delete(" ") + delete_optional_space = pynini.closure(pynutil.delete(" "), 0, 1) + delete_comma_sep = delete_comma + delete_optional_space + + day_num_padded = pynini.union( + days, + teens_and_ties, + ) + + day_num_bare = pynini.union( + pynini.compose(pad_latin, days), + pynini.compose(pad_devanagari, days), + ) + + days_graph_padded = pynutil.insert("day: \"") + day_num_padded + pynutil.insert("\"") + insert_space + days_graph_bare = pynutil.insert("day: \"") + day_num_bare + pynutil.insert("\"") + insert_space + + month_name_acceptor = pynini.project(months, "output") + + months_numeric_padded = months + + months_numeric_bare = pynini.union( + pynini.compose(pad_latin, months), + pynini.compose(pad_devanagari, months), + ) + + months_graph_numeric_padded = ( + pynutil.insert("month: \"") + months_numeric_padded + pynutil.insert("\"") + insert_space + ) + + months_fst_padded = pynini.union(months_numeric_padded, month_name_acceptor) + months_graph_padded = pynutil.insert("month: \"") + months_fst_padded + pynutil.insert("\"") + insert_space + + months_fst_bare = pynini.union(months_numeric_bare, month_name_acceptor) + months_graph_bare = pynutil.insert("month: \"") + months_fst_bare + pynutil.insert("\"") + insert_space + + month_name_graph = pynutil.insert("month: \"") + month_name_acceptor + pynutil.insert("\"") + insert_space + + years_graph = pynutil.insert("year: \"") + graph_year + pynutil.insert("\"") + insert_space + + era_graph = pynutil.insert("era: \"") + year_suffix + pynutil.insert("\"") + insert_space + + range_graph = pynini.cross("-", "से") + + century_number = pynini.compose(pynini.closure(NEMO_ALL_DIGIT, 1), cardinal_graph) + pynini.accep("वीं") + century_text = pynutil.insert("era: \"") + century_number + pynutil.insert("\"") + insert_space + + year_number = graph_year + suffix_union + year_text = pynutil.insert("era: \"") + year_number + pynutil.insert("\"") + insert_space + + year_prefix = pynutil.insert("era: \"") + prefix_union + pynini.accep(" ") + graph_year + pynutil.insert("\"") + + year_prefix_suffix = ( + pynutil.insert("era: \"") + + prefix_union + + pynini.accep(" ") + + graph_year + + suffix_union + + pynutil.insert("\"") + ) + + graph_verbalized_year_suffix = ( + pynutil.insert("era: \"") + verbalized_year_sou + suffix_union + pynutil.insert("\"") + insert_space + ) + + graph_verbalized_year_bare = ( + pynutil.insert("era: \"") + verbalized_year_sou + pynutil.insert("\"") + insert_space + ) + + graph_verbalized_year_prefix = ( + pynutil.insert("era: \"") + prefix_union + pynini.accep(" ") + verbalized_year_sou + pynutil.insert("\"") + ) + + graph_verbalized_year_prefix_suffix = ( + pynutil.insert("era: \"") + + prefix_union + + pynini.accep(" ") + + verbalized_year_sou + + suffix_union + + pynutil.insert("\"") + ) + + graph_dd_mm = days_graph_padded + delete_dash + months_graph_padded + + graph_d_m = days_graph_bare + delete_dash + months_graph_bare + + graph_dd_mm_yyyy = days_graph_padded + delete_dash + months_graph_padded + delete_dash + years_graph + + graph_d_m_yyyy = days_graph_bare + delete_dash + months_graph_bare + delete_dash + years_graph + + graph_dd_month = days_graph_padded + delete_space + months_graph_numeric_padded + + graph_dd_month_comma_yyyy = ( + days_graph_padded + delete_space + months_graph_padded + delete_comma_sep + years_graph + ) + + graph_dd_month_comma_yyyy_era = ( + days_graph_padded + delete_space + months_graph_padded + delete_comma_sep + years_graph + era_graph + ) + + graph_month_comma_yyyy = months_graph_padded + delete_comma_sep + years_graph + + graph_month_comma_yyyy_era = months_graph_padded + delete_comma_sep + years_graph + era_graph + + graph_month_name_yyyy = month_name_graph + delete_space + years_graph + + graph_year_era_only = ( + pynutil.insert("era: \"") + + graph_year_era + + insert_space + + year_suffix + + pynutil.insert("\"") + + insert_space + ) + + graph_range = ( + pynutil.insert("era: \"") + + cardinal_graph + + insert_space + + range_graph + + insert_space + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" preserve_order: true ") + ) + + graph_year_suffix = era_graph + + final_graph = ( + pynutil.add_weight(graph_dd_month_comma_yyyy_era, -0.003) + | pynutil.add_weight(graph_month_comma_yyyy_era, -0.003) + | pynutil.add_weight(graph_dd_mm_yyyy, -0.001) + | pynutil.add_weight(graph_d_m_yyyy, -0.001) + | pynutil.add_weight(graph_dd_month_comma_yyyy, -0.001) + | pynutil.add_weight(graph_dd_mm, -0.001) + | pynutil.add_weight(graph_d_m, -0.001) + | pynutil.add_weight(graph_dd_month, -0.001) + | pynutil.add_weight(graph_month_name_yyyy, -0.2) + | pynutil.add_weight(graph_month_comma_yyyy, -0.2) + | pynutil.add_weight(graph_year_era_only, -0.005) + | pynutil.add_weight(graph_range, -0.005) + | pynutil.add_weight(graph_year_suffix, -0.001) + | pynutil.add_weight(century_text, -0.001) + | pynutil.add_weight(graph_verbalized_year_prefix_suffix, -0.012) + | pynutil.add_weight(graph_verbalized_year_prefix, -0.011) + | pynutil.add_weight(graph_verbalized_year_suffix, -0.010) + | pynutil.add_weight(graph_verbalized_year_bare, -0.009) + | pynutil.add_weight(year_prefix_suffix, -0.010) + | pynutil.add_weight(year_prefix, -0.009) + | pynutil.add_weight(year_text, -0.001) + ) + + self.final_graph = final_graph.optimize() + self.fst = self.add_tokens(self.final_graph) diff --git a/nemo_text_processing/text_normalization/hi/taggers/decimal.py b/nemo_text_processing/text_normalization/hi/taggers/decimal.py new file mode 100644 index 000000000..7522de2bb --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/decimal.py @@ -0,0 +1,84 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + +quantities = pynini.string_file(get_abs_path("data/numbers/thousands.tsv")) + + +def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstLike') -> 'pynini.FstLike': + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. १ लाख -> integer_part: "एक" quantity: "लाख" + e.g. १.५ लाख -> integer_part: "एक" fractional_part: "पाँच" quantity: "लाख" + + Args: + decimal: decimal FST + cardinal_up_to_hundred: cardinal FST + """ + numbers = cardinal_up_to_hundred + + res = ( + pynutil.insert("integer_part: \"") + + numbers + + pynutil.insert("\"") + + insert_space + + pynutil.insert("quantity: \"") + + quantities + + pynutil.insert("\"") + ) + res |= decimal + insert_space + pynutil.insert("quantity: \"") + quantities + pynutil.insert("\"") + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + -१२.५००६ अरब -> decimal { negative: "true" integer_part: "बारह" fractional_part: "पाँच शून्य शून्य छह" quantity: "अरब" } + १ अरब -> decimal { integer_part: "एक" quantity: "अरब" } + + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + graph_digit = cardinal.digit | cardinal.zero + cardinal_graph = cardinal.graph_without_leading_zeros + + self.graph = graph_digit + pynini.closure(insert_space + graph_digit).optimize() + + point = pynutil.delete(".") + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + insert_space, + 0, + 1, + ) + + self.graph_fractional = pynutil.insert("fractional_part: \"") + self.graph + pynutil.insert("\"") + self.graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + + final_graph_wo_sign = self.graph_integer + point + insert_space + self.graph_fractional + + self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(final_graph_wo_sign, cardinal_graph) + + final_graph = optional_graph_negative + self.final_graph_wo_negative + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/electronic.py b/nemo_text_processing/text_normalization/hi/taggers/electronic.py new file mode 100644 index 000000000..e1b93835e --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/electronic.py @@ -0,0 +1,181 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_ALPHA, NEMO_DIGIT, NEMO_HI_DIGIT, GraphFst +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic: as URLs, email addresses, file paths, + IP addresses, domains, and chemical formulas. + e.g. kumar@gmail.com -> tokens { electronic { username: "kumar" domain: "gmail.com" } } + e.g. https://google.com/ -> tokens { electronic { protocol: "https" domain: "google.com/" } } + e.g. C:\\Users\\HP\\Desktop -> tokens { electronic { path: "C:\\Users\\HP\\Desktop" } } + e.g. 192.168.1.1 -> tokens { electronic { domain: "192.168.1.1" } } + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + subscript_digit = pynini.project( + pynini.string_file(get_abs_path("data/electronic/subscript_digit.tsv")), "input" + ) + alphanumeric = NEMO_ALPHA | NEMO_DIGIT | NEMO_HI_DIGIT | subscript_digit + + symbol_dict = {"email": [], "url": [], "unix": [], "windows": [], "chem": []} + + with open(get_abs_path("data/electronic/symbol_classes.tsv"), "r", encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + parts = line.strip().split("\t") + if len(parts) == 2: + sym = parts[0] + classes = parts[1].split(",") + for c in classes: + if c in symbol_dict: + symbol_dict[c].append(sym) + + email_symbols = pynini.union(*symbol_dict["email"]) + url_symbols = pynini.union(*symbol_dict["url"]) + unix_symbols = pynini.union(*symbol_dict["unix"]) + win_symbols = pynini.union(*symbol_dict["windows"]) + chemical_symbols = pynini.union(*symbol_dict["chem"]) + + unix_segment_syms = pynini.union(*[s for s in symbol_dict["unix"] if s != "/"]) + + username_chars = NEMO_ALPHA | NEMO_DIGIT | email_symbols + username = pynutil.insert("username: \"") + pynini.closure(username_chars, 1) + pynutil.insert("\"") + + domain_chars = NEMO_ALPHA | NEMO_DIGIT | pynini.accep(".") | pynini.accep("-") + domain = pynutil.insert(" domain: \"") + pynini.closure(domain_chars, 1) + pynutil.insert("\"") + + email_graph = username + pynini.cross("@", "") + domain + + protocol_start = pynini.cross("https://", "https") | pynini.cross("http://", "http") + protocol_end = pynini.cross("www.", "www") + protocol = ( + pynutil.insert("protocol: \"") + + ( + pynutil.add_weight(protocol_start + protocol_end, 1.0) + | pynutil.add_weight(protocol_start, 1.01) + | pynutil.add_weight(protocol_end, 1.02) + ) + + pynutil.insert("\"") + ) + + url_path_chars = alphanumeric | url_symbols + url_path = pynini.closure(url_path_chars, 1) + url_domain = pynutil.insert(" domain: \"") + url_path + pynutil.insert("\"") + url_graph = protocol + url_domain + + drive_letter = NEMO_ALPHA + windows_path_chars = alphanumeric | win_symbols | pynini.accep(" ") + windows_path = ( + pynutil.insert("path: \"") + + drive_letter + + pynini.accep(":") + + pynini.accep("\\") + + pynini.closure(windows_path_chars, 1) + + pynutil.insert("\"") + ) + + unix_path_chars = alphanumeric | unix_symbols + unix_segment_chars = alphanumeric | unix_segment_syms + unix_segment = pynini.closure(unix_segment_chars, 1) + + abs_unix_path = pynini.accep("/") + pynini.closure(unix_path_chars, 1) + rel_unix_path = unix_segment + pynini.accep("/") + pynini.closure(unix_path_chars, 0) + + unix_path = pynutil.insert("path: \"") + (abs_unix_path | rel_unix_path) + pynutil.insert("\"") + + backslash_path_chars = alphanumeric | unix_segment_syms | pynini.accep("\\") | pynini.accep(" ") + backslash_path = ( + pynutil.insert("path: \"") + + pynini.accep("\\") + + pynini.closure(backslash_path_chars, 1) + + pynutil.insert("\"") + ) + + ip_octet = pynini.closure(NEMO_DIGIT, 1, 3) + dot_octet = pynini.accep(".") + ip_octet + ip_address = pynutil.insert("domain: \"") + ip_octet + pynini.closure(dot_octet, 3, 3) + pynutil.insert("\"") + + domain_segment_chars = NEMO_ALPHA | NEMO_DIGIT | pynini.accep("-") + domain_segment = pynini.closure(domain_segment_chars, 1) + + tld = pynini.project(pynini.string_file(get_abs_path("data/electronic/domain.tsv")), "input") + + domain_body = ( + pynini.closure(domain_segment + pynini.accep("."), 1) + tld + pynini.closure(pynini.accep(".") + tld, 0, 1) + ) + + combined_domain = ( + pynutil.insert("domain: \"") + domain_body + pynini.closure(pynini.accep("/"), 0, 1) + pynutil.insert("\"") + ) + + known_extensions = pynini.project( + pynini.string_file(get_abs_path("data/electronic/file_extensions.tsv")), "input" + ) + filename_stem_chars = NEMO_ALPHA | NEMO_DIGIT | pynini.accep("-") | pynini.accep("_") + filename_stem = pynini.closure(filename_stem_chars, 1) + file_with_extension = ( + pynutil.insert("domain: \"") + filename_stem + pynini.accep(".") + known_extensions + pynutil.insert("\"") + ) + + elements = pynini.project(pynini.string_file(get_abs_path("data/electronic/elements.tsv")), "input") + + chem_number = pynini.closure(NEMO_DIGIT | subscript_digit, 1) + + chem_block = elements + pynini.closure(chem_number, 0, 1) + + chem_sequence_chars = chem_block | chemical_symbols | chem_number + + raw_chemical = pynini.closure(chemical_symbols) + chem_block + pynini.closure(chem_sequence_chars) + + any_chem = pynini.closure(chem_sequence_chars) + has_open = any_chem + pynini.accep("(") + any_chem + no_open = pynini.difference(any_chem, has_open) + ends_with_close = any_chem + pynini.accep(")") + + unbalanced_trailing = pynini.intersect(no_open, ends_with_close) + valid_chemical = pynini.difference(raw_chemical, unbalanced_trailing).optimize() + + # Recognise a chemical formula only when it uses subscript notation + chem_sigma = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | subscript_digit | chemical_symbols) + contains_subscript = chem_sigma + subscript_digit + chem_sigma + valid_chemical = pynini.intersect(valid_chemical, contains_subscript).optimize() + + # Chemical formulas carry a dedicated tag so the verbalizer can spell element + chemical_formula = pynutil.insert("fragment_id: \"") + valid_chemical + pynutil.insert("\"") + + graph = ( + pynutil.add_weight(url_graph, 1.0) + | pynutil.add_weight(email_graph, 1.0) + | pynutil.add_weight(windows_path, 1.0) + | pynutil.add_weight(unix_path, 1.0) + | pynutil.add_weight(backslash_path, 1.0) + | pynutil.add_weight(ip_address, 1.0) + | pynutil.add_weight(combined_domain, 1.1) + | pynutil.add_weight(file_with_extension, 1.1) + | pynutil.add_weight(chemical_formula, 1.2) + ) + + self.graph = graph.optimize() + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/fraction.py b/nemo_text_processing/text_normalization/hi/taggers/fraction.py new file mode 100644 index 000000000..8b72b25b2 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/fraction.py @@ -0,0 +1,140 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + HI_DEDH, + HI_DHAI, + HI_PAUNE, + HI_SADHE, + HI_SAVVA, + NEMO_SPACE, + GraphFst, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + +HI_ONE_HALF = "१/२" +HI_ONE_QUARTER = "१/४" +HI_THREE_QUARTERS = "३/४" + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction + "२३ ४/६" -> + fraction { integer: "तेईस" numerator: "चार" denominator: "छह"} + ४/६" -> + fraction { numerator: "चार" denominator: "छह"} + + + Args: + cardinal: cardinal GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.final_graph + + self.optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + pynutil.insert(NEMO_SPACE), 0, 1 + ) + + self.integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + + self.numerator = ( + pynutil.insert("numerator: \"") + + cardinal_graph + + pynini.cross(pynini.union("/", NEMO_SPACE + "/" + NEMO_SPACE), "\"") + + pynutil.insert(NEMO_SPACE) + ) + + self.denominator = pynutil.insert("denominator: \"") + cardinal_graph + pynutil.insert("\"") + + dedh_dhai_graph = pynini.string_map( + [("१" + NEMO_SPACE + HI_ONE_HALF, HI_DEDH), ("२" + NEMO_SPACE + HI_ONE_HALF, HI_DHAI)] + ) + + savva_numbers = cardinal_graph + pynini.cross(NEMO_SPACE + HI_ONE_QUARTER, "") + savva_graph = pynutil.insert(HI_SAVVA) + pynutil.insert(NEMO_SPACE) + savva_numbers + + sadhe_numbers = cardinal_graph + pynini.cross(NEMO_SPACE + HI_ONE_HALF, "") + sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers + + paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) + paune_numbers = paune + pynini.cross(NEMO_SPACE + HI_THREE_QUARTERS, "") + paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers + + common_fraction_map = pynini.string_file(get_abs_path("data/fraction/common_fractions.tsv")) + + graph_common_fraction = ( + pynutil.insert("morphosyntactic_features: \"") + + common_fraction_map + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + graph_dedh_dhai = ( + pynutil.insert("morphosyntactic_features: \"") + + dedh_dhai_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + graph_savva = ( + pynutil.insert("morphosyntactic_features: \"") + + savva_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + graph_sadhe = ( + pynutil.insert("morphosyntactic_features: \"") + + sadhe_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + graph_paune = ( + pynutil.insert("morphosyntactic_features: \"") + + paune_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + final_graph = ( + self.optional_graph_negative + + pynini.closure(self.integer + pynini.accep(NEMO_SPACE), 0, 1) + + self.numerator + + self.denominator + ) + + weighted_graph = ( + final_graph + | pynutil.add_weight(graph_common_fraction, -0.3) + | pynutil.add_weight(graph_dedh_dhai, -0.2) + | pynutil.add_weight(graph_paune, -0.2) + | pynutil.add_weight(graph_savva, -0.1) + | pynutil.add_weight(graph_sadhe, -0.1) + ) + + self.graph = weighted_graph + + graph = self.graph + graph = self.add_tokens(graph) + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py new file mode 100644 index 000000000..67043b727 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -0,0 +1,477 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + ASTERISK, + COMMA, + DECIMAL_25, + DECIMAL_75, + HI_BY, + HI_DEDH, + HI_DHAI, + HI_PAUNE, + HI_PERIOD, + HI_SADHE, + HI_SAVVA, + HYPHEN, + LOWERCASE_X, + MIN_NEG_WEIGHT, + NEMO_CHAR, + NEMO_DIGIT, + NEMO_HI_DIGIT, + NEMO_NOT_SPACE, + NEMO_SPACE, + NEMO_WHITE_SPACE, + ONE_POINT_FIVE, + PERIOD, + POINT_FIVE, + SLASH, + TWO_POINT_FIVE, + UPPERCASE_X, + GraphFst, + capitalized_input_graph, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + +digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + +# Shared Address Maps +zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) +telephone_number = pynini.string_file(get_abs_path("data/telephone/number.tsv")) +states_map = pynini.string_file(get_abs_path("data/address/states.tsv")) +cities_map = pynini.string_file(get_abs_path("data/address/cities.tsv")) +special_characters_map = pynini.string_file(get_abs_path("data/address/special_characters.tsv")) +letters_map = pynini.string_file(get_abs_path("data/address/letters.tsv")) +context_map = pynini.string_file(get_abs_path("data/address/context.tsv")) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure, suppletive aware, e.g. + -१२kg -> measure { negative: "true" cardinal { integer: "बारह" } units: "किलोग्राम" } + -१२.२kg -> measure { decimal { negative: "true" integer_part: "बारह" fractional_part: "दो"} units: "किलोग्राम" } + मुंबई ८८४४०४ -> measure { units: "address" cardinal { integer: "मुंबई आठ आठ चार चार शून्य चार" } preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def get_structured_address_graph(self, cardinal: GraphFst, ordinal: GraphFst, input_case: str): + """ + Minimal address tagger for state/city + pincode patterns only. + + Examples: + "मुंबई ८८४४०४" -> "मुंबई आठ आठ चार चार शून्य चार" + "गोवा १२३४५६" -> "गोवा एक दो तीन चार पाँच छह" + "100 फीट रोड, चेन्नई" -> "एक सौ फीट रोड, चेन्नई" + """ + # State/city keywords + + state_city_names = pynini.union(states_map, cities_map).optimize() + + # Digit mappings (shared maps loaded once at module level) + num_token = (digit | zero | telephone_number).optimize() + + # Pincode (6 digits) -> always digit-by-digit (length >= 4) + pincode = (num_token + pynini.closure(insert_space + num_token, 5, 5)).optimize() + + # Street number: Use centralized digit-by-digit logic from cardinal + street_num = cardinal.code_num_graph + + # Text: words with trailing separator (comma? + space) + any_digit = pynini.union(NEMO_HI_DIGIT, NEMO_DIGIT).optimize() + punctuation = pynini.union(COMMA, PERIOD, HI_PERIOD).optimize() + word_char = pynini.difference(NEMO_NOT_SPACE, pynini.union(any_digit, punctuation)).optimize() + word = pynini.closure(word_char, 1) + + # Separator: optional comma followed by mandatory space + sep = pynini.closure(pynini.accep(COMMA), 0, 1) + pynini.accep(NEMO_SPACE) + word_with_sep = word + sep + # Consume inline address numbers using the same 1-3 (cardinal) / 4+ (digit-by-digit) rule + num_with_sep = street_num + sep + text = pynini.closure(pynini.union(word_with_sep, num_with_sep), 0, 5).optimize() + + # Pattern: [street_num + sep]? text state/city [space pincode] + pattern = ( + pynini.closure(street_num + sep, 0, 1) + + text + + state_city_names + + pynini.closure(pynini.accep(NEMO_SPACE) + pincode, 0, 1) + ).optimize() + + graph = ( + pynutil.insert('units: "address" cardinal { integer: "') + + pattern + + pynutil.insert('" } preserve_order: true') + ) + return pynutil.add_weight(graph, 1.0).optimize() + + def get_address_graph(self, cardinal: GraphFst, ordinal: GraphFst, serial: GraphFst, input_case: str): + """ + Address tagger that fires when address context keywords are present. + + Examples: + "७०० ओक स्ट्रीट" -> "सात सौ ओक स्ट्रीट" + "६६-४ पार्क रोड" -> "छियासठ हाइफ़न चार पार्क रोड" + "593988" (6-digit pincode) -> "पाँच नौ तीन नौ आठ आठ" + "32A नाज़ प्लाज़ा" -> "बत्तीस ए नाज़ प्लाज़ा" + """ + # Retain internal weights of ordinal graph + ordinal_graph = ordinal.graph + # Alphanumeric to word mappings (digits, special characters, telephone digits) + char_to_word = (digit | zero | special_characters_map | telephone_number).optimize() + letter_to_word = capitalized_input_graph(letters_map) + # Identity acceptor for keywords (Devanagari/English) to prevent unintended rewrites/transliteration + address_keywords = pynini.project( + capitalized_input_graph(context_map), + "input", + ).optimize() + + # Alphanumeric processing: treat digits, letters, and -/ as convertible tokens + single_digit = pynini.union(NEMO_DIGIT, NEMO_HI_DIGIT).optimize() + special_chars = pynini.union(HYPHEN, SLASH).optimize() + single_letter = pynini.project(letter_to_word, "input").optimize() + convertible_char = pynini.union(single_digit, special_chars, single_letter) + non_space_char = pynini.difference( + NEMO_CHAR, pynini.union(NEMO_WHITE_SPACE, convertible_char, pynini.accep(COMMA)) + ).optimize() + + comma_processor = delete_space + pynini.accep(COMMA) + ordinal_processor = insert_space + ordinal_graph + latin_word = single_letter + pynini.closure(single_letter, 1) + english_word_processor = insert_space + latin_word + letter_processor = insert_space + pynini.compose(single_letter, letter_to_word) + special_char_processor = insert_space + pynini.compose(special_chars, char_to_word) + other_word_processor = insert_space + pynini.closure(non_space_char, 1) + + code_processor = insert_space + serial.mixed_alphanum_graph + + # Pure numeric runs: 1-3 digits read as cardinal, 4+ digits read digit-by-digit + number_run = pynini.compose(pynini.closure(single_digit, 1), cardinal.code_num_graph).optimize() + + # A tiny positive penalty prevents multiple uses of this arc, forcing the FST to consume contiguous digits as ONE run instead of aggressively splitting them. + number_run_processor = insert_space + number_run + + token_processor = ( + pynini.accep(NEMO_SPACE) + | comma_processor + | special_char_processor + | code_processor + | pynutil.add_weight(ordinal_processor, MIN_NEG_WEIGHT) + | pynutil.add_weight(number_run_processor, 0.5) # Keeps numbers together + | letter_processor + | english_word_processor + | pynutil.add_weight(other_word_processor, 0.1) # Keeps Hindi words together + ).optimize() + + full_string_processor = pynini.closure(token_processor, 1).optimize() + + # Window-based context matching around address keywords for robust detection + word_boundary = pynini.union( + NEMO_WHITE_SPACE, pynini.accep(COMMA), pynini.accep(HI_PERIOD), pynini.accep(PERIOD) + ).optimize() + non_boundary_char = pynini.difference(NEMO_CHAR, word_boundary) + word = pynini.closure(non_boundary_char, 1).optimize() + word_with_boundary = word + pynini.closure(word_boundary) + window = pynini.closure(word_with_boundary, 0, 5).optimize() + boundary = pynini.closure(word_boundary, 1).optimize() + input_pattern = pynini.union( + address_keywords + boundary + window, + window + boundary + address_keywords + pynini.closure(boundary + window, 0, 1), + ).optimize() + address_graph = pynini.compose(input_pattern, full_string_processor).optimize() + graph = ( + pynutil.insert('units: "address" cardinal { integer: "') + + address_graph + + pynutil.insert('" } preserve_order: true') + ) + return graph.optimize() + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, serial: GraphFst, input_case: str): + super().__init__(name="measure", kind="classify") + + cardinal_graph = ( + cardinal.zero + | cardinal.digit + | cardinal.teens_and_ties + | cardinal.graph_hundreds + | cardinal.graph_thousands + | cardinal.graph_ten_thousands + | cardinal.graph_lakhs + | cardinal.graph_ten_lakhs + ) + point = pynutil.delete(".") + decimal_integers = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + decimal_graph = decimal_integers + point + insert_space + decimal.graph_fractional + unit_graph = pynini.string_file(get_abs_path("data/measure/unit.tsv")) + + # Year unit variants for formal/informal handling + year_informal = pynini.string_map([("yr", "साल")]) + year_formal = pynini.string_file(get_abs_path("data/measure/unit_year_formal.tsv")) + + # All units EXCEPT year + unit_inputs_except_yr = pynini.difference(pynini.project(unit_graph, "input"), pynini.accep("yr")) + unit_graph_no_year = pynini.compose(unit_inputs_except_yr, unit_graph) + + # Load quarterly units from separate files: map (FST) and list (FSA) + quarterly_units_map = pynini.string_file(get_abs_path("data/measure/quarterly_units_map.tsv")) + quarterly_units_list = pynini.string_file(get_abs_path("data/measure/quarterly_units_list.tsv")) + quarterly_units_graph = pynini.union(quarterly_units_map, quarterly_units_list) + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + insert_space, + 0, + 1, + ) + + # Define the quarterly measurements - support both Devanagari and Arabic digits + quarter = pynini.union( + pynini.cross(POINT_FIVE, HI_SADHE), + pynini.cross(ONE_POINT_FIVE, HI_DEDH), + pynini.cross(TWO_POINT_FIVE, HI_DHAI), + ) + quarter_graph = pynutil.insert("integer_part: \"") + quarter + pynutil.insert("\"") + + # Define the unit handling + unit = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + unit_graph_no_year + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + units = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + quarterly_units_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + # Year-specific unit wrappers + unit_year_informal = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + year_informal + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + unit_year_formal = ( + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + year_formal + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + # Cardinal >= 1000 -> formal year (वर्ष) + # Use graph_without_leading_zeros which covers all number ranges (thousands to shankhs) + cardinal_large = cardinal.graph_without_leading_zeros + + # Cardinal < 1000 -> informal year (साल) + cardinal_small = cardinal.zero | cardinal.digit | cardinal.teens_and_ties | cardinal.graph_hundreds + + symbol_graph = pynini.string_map( + [ + (LOWERCASE_X, HI_BY), + (UPPERCASE_X, HI_BY), + (ASTERISK, HI_BY), + ] + ) + + graph_decimal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal_graph + + pynutil.insert(" }") + + delete_space + + unit + ) + + # Support both Devanagari and Arabic digits for dedh/dhai patterns + dedh_dhai = pynini.union( + pynini.cross(ONE_POINT_FIVE, HI_DEDH), + pynini.cross(TWO_POINT_FIVE, HI_DHAI), + ) + dedh_dhai_graph = pynutil.insert("integer: \"") + dedh_dhai + pynutil.insert("\"") + + # Support both Devanagari and Arabic digits for savva pattern + savva_numbers = cardinal_graph + pynini.cross(DECIMAL_25, "") + savva_graph = ( + pynutil.insert("integer: \"") + + pynutil.insert(HI_SAVVA) + + pynutil.insert(NEMO_SPACE) + + savva_numbers + + pynutil.insert("\"") + ) + + # Support both Devanagari and Arabic digits for sadhe pattern + sadhe_numbers = cardinal_graph + pynini.cross(POINT_FIVE, "") + sadhe_graph = ( + pynutil.insert("integer: \"") + + pynutil.insert(HI_SADHE) + + pynutil.insert(NEMO_SPACE) + + sadhe_numbers + + pynutil.insert("\"") + ) + + paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) + # Support both Devanagari and Arabic digits for paune pattern + paune_numbers = paune + pynini.cross(DECIMAL_75, "") + paune_graph = ( + pynutil.insert("integer: \"") + + pynutil.insert(HI_PAUNE) + + pynutil.insert(NEMO_SPACE) + + paune_numbers + + pynutil.insert("\"") + ) + + graph_dedh_dhai = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + dedh_dhai_graph + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + delete_space + + units + ) + + graph_savva = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + savva_graph + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + delete_space + + units + ) + + graph_sadhe = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + sadhe_graph + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + delete_space + + units + ) + + graph_paune = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + paune_graph + + pynutil.insert(" }") + + delete_space + + units + ) + + graph_cardinal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + delete_space + + unit + ) + + # Large numbers (>=1000) + yr -> formal (वर्ष) + graph_cardinal_year_formal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_large + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + delete_space + + unit_year_formal + ) + + # Small numbers (<1000) + yr -> informal (साल) + graph_cardinal_year_informal = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_small + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("}") + + delete_space + + unit_year_informal + ) + + # Regular decimals (e.g., 16.07) + yr -> formal (वर्ष) + graph_decimal_year_formal = ( + pynutil.insert("decimal { ") + + optional_graph_negative + + decimal_graph + + pynutil.insert(" }") + + delete_space + + unit_year_formal + ) + + # Handling cardinal clubbed with symbol as single token + graph_exceptions = ( + pynutil.insert("cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("units: \"") + + symbol_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("} }") + + insert_space + + pynutil.insert("tokens { cardinal { ") + + optional_graph_negative + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + ) + + address_graph = self.get_address_graph(cardinal, ordinal, serial, input_case) + structured_address_graph = self.get_structured_address_graph(cardinal, ordinal, input_case) + + graph = ( + pynutil.add_weight(graph_decimal, 0.1) + | pynutil.add_weight(graph_decimal_year_formal, 0.1) + | pynutil.add_weight(graph_cardinal, 0.1) + | pynutil.add_weight(graph_cardinal_year_formal, 0.1) + | pynutil.add_weight(graph_cardinal_year_informal, -0.1) # Higher priority for small numbers + | pynutil.add_weight(graph_exceptions, 0.1) + | pynutil.add_weight(graph_dedh_dhai, -0.2) + | pynutil.add_weight(graph_savva, -0.1) + | pynutil.add_weight(graph_sadhe, -0.1) + | pynutil.add_weight(graph_paune, -0.5) + | address_graph + | structured_address_graph + ) + self.graph = graph.optimize() + + final_graph = self.add_tokens(graph) + self.fst = final_graph diff --git a/nemo_text_processing/text_normalization/hi/taggers/money.py b/nemo_text_processing/text_normalization/hi/taggers/money.py new file mode 100644 index 000000000..16f389ae7 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/money.py @@ -0,0 +1,158 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.hi.utils import get_abs_path, load_labels + +currency_graph = pynini.string_file(get_abs_path("data/money/currency.tsv")) +currency_singular_graph = pynini.string_file(get_abs_path("data/money/currency_singular.tsv")) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, suppletive aware, e.g. + ₹५० -> money { currency_maj: "रुपए" integer_part: "पचास" } + ₹५०.५० -> money { currency_maj: "रुपए" integer_part: "पचास" fractional_part: "पचास" currency_min: "पैसे" } + ₹०.५० -> { money { currency_maj: "रुपए" integer_part: "शून्य" fractional_part: "पचास" currency_min: "पैसे" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="money", kind="classify") + + cardinal_graph = cardinal.final_graph + + _en_to_hi_digit = pynini.string_file(get_abs_path("data/ordinal/en_to_hi_digit.tsv")) + _deva_to_ascii = pynini.invert(_en_to_hi_digit) + deva_to_ascii = pynini.closure(_deva_to_ascii | pynini.union(*"0123456789"), 1) + + _ascii_digit = pynini.union(*"0123456789") + _ascii_nonzero = pynini.union(*"123456789") + _deva_nonzero = pynini.union(*"१२३४५६७८९") + _any_digit = _ascii_digit | pynini.union(*"०१२३४५६७८९") + _any_nonzero = _ascii_nonzero | _deva_nonzero + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", '"true"') + insert_space, + 0, + 1, + ) + + currency_major = pynutil.insert('currency_maj: "') + currency_graph + pynutil.insert('"') + currency_major_singular = pynutil.insert('currency_maj: "') + currency_singular_graph + pynutil.insert('"') + + one = pynini.union("1", "१") + integer_one = pynutil.insert('integer_part: "') + (one @ cardinal_graph) + pynutil.insert('"') + integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + + strip_trailing_zeros = pynini.closure(_ascii_digit) + _ascii_nonzero + pynini.closure(pynutil.delete("0")) + canonicalise = ( + (pynutil.delete("0") + _ascii_nonzero) + | (_ascii_nonzero + pynutil.insert("0")) + | (_ascii_nonzero + _ascii_digit) + ) + two_digits_fractional_part = deva_to_ascii @ strip_trailing_zeros @ canonicalise + + fraction = ( + pynutil.insert('fractional_part: "') + (two_digits_fractional_part @ cardinal_graph) + pynutil.insert('"') + ) + + optional_delete_fractional_zeros = pynini.closure( + pynutil.delete(".") + pynini.closure(pynutil.delete("0") | pynutil.delete("०"), 1), + 0, + 1, + ) + + has_3plus_sig_digits = _any_digit + _any_digit + _any_nonzero + pynini.closure(_any_digit) + single_digit = _any_digit @ cardinal.single_digits_graph + decimal_digits = ( + pynutil.insert('fractional_part: "') + + single_digit + + pynini.closure(insert_space + single_digit) + + pynutil.insert('"') + ) + guarded_decimal_digits = has_3plus_sig_digits @ decimal_digits + + graph_decimal_path = ( + optional_graph_negative + + currency_major + + insert_space + + pynutil.insert('integer_part: "') + + cardinal_graph + + pynutil.insert('"') + + pynini.cross(".", " ") + + guarded_decimal_digits + ).optimize() + + graph_major_only_singular = ( + optional_graph_negative + + currency_major_singular + + insert_space + + integer_one + + optional_delete_fractional_zeros + ).optimize() + + graph_major_only = ( + optional_graph_negative + currency_major + insert_space + integer + optional_delete_fractional_zeros + ).optimize() + + maj_labels = load_labels(get_abs_path("data/money/currency.tsv")) + maj_singular_labels = load_labels(get_abs_path("data/money/currency_singular.tsv")) + maj_to_min = dict(load_labels(get_abs_path("data/money/major_minor_currencies.tsv"))) + + def _build_major_and_minor(sym_maj_labels, int_graph): + result = None + for sym, maj in sym_maj_labels: + min_name = maj_to_min.get(maj) + if not min_name: + continue + + curr_maj = pynutil.insert('currency_maj: "') + pynini.cross(sym, maj) + pynutil.insert('"') + curr_min = pynutil.insert('currency_min: "') + pynutil.insert(min_name) + pynutil.insert('"') + + g = ( + optional_graph_negative + + curr_maj + + insert_space + + int_graph + + pynini.cross(".", " ") + + fraction + + insert_space + + curr_min + ).optimize() + + result = g if result is None else pynini.union(result, g).optimize() + + return result + + graph_major_and_minor = _build_major_and_minor(maj_labels, integer) + graph_major_and_minor_singular = _build_major_and_minor(maj_singular_labels, integer_one) + + graph_currencies = ( + pynutil.add_weight(graph_major_only_singular | graph_major_and_minor_singular, -0.001) + | pynutil.add_weight(graph_decimal_path, -0.0005) + | graph_major_only + | graph_major_and_minor + ) + + graph = graph_currencies.optimize() + self.fst = self.add_tokens(graph) diff --git a/nemo_text_processing/text_normalization/hi/taggers/ordinal.py b/nemo_text_processing/text_normalization/hi/taggers/ordinal.py new file mode 100644 index 000000000..bef7ba426 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/ordinal.py @@ -0,0 +1,61 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst +from nemo_text_processing.text_normalization.hi.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying Hindi ordinals, e.g. + १०वां -> ordinal { integer: "दसवां" } + 12वीं -> ordinal { integer: "इक्कीसवीं" } # English/arabic digits also supported + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: CardinalFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + suffixes_list = pynini.string_file(get_abs_path("data/ordinal/suffixes.tsv")) + suffixes_map = pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")) + suffixes_fst = pynini.union(suffixes_list, suffixes_map) + exceptions = pynini.string_file(get_abs_path("data/ordinal/exceptions.tsv")) + + # Limit cardinal graph to thousands range for faster compilation + limited_cardinal_graph = ( + cardinal.digit + | cardinal.zero + | cardinal.teens_and_ties + | cardinal.graph_hundreds + | cardinal.graph_thousands + | cardinal.graph_ten_thousands + ).optimize() + + graph = limited_cardinal_graph + suffixes_fst + exceptions = pynutil.add_weight(exceptions, -0.1) + graph = pynini.union(exceptions, graph) + + self.graph = graph.optimize() + + final_graph = pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/punctuation.py b/nemo_text_processing/text_normalization/hi/taggers/punctuation.py new file mode 100644 index 000000000..ea9cfc72e --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/punctuation.py @@ -0,0 +1,62 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + self.punct_marks = [p for p in punct_unicode + list(s)] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + pynini.union( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)), + (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)), + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/roman.py b/nemo_text_processing/text_normalization/hi/taggers/roman.py new file mode 100644 index 000000000..ea8d259fe --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/roman.py @@ -0,0 +1,138 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst, convert_space, insert_space +from nemo_text_processing.text_normalization.hi.utils import get_abs_path, load_labels + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying Roman numerals in Hindi text. + e.g. भास्कर-II -> tokens { roman { key_cardinal: "भास्कर" integer: "II" } } + e.g. कक्षा XII -> tokens { roman { key_cardinal: "कक्षा" integer: "XII" } } + e.g. XIIवीं कक्षा -> tokens { roman { integer: "XII" default_ordinal: "बारहवीं" key_cardinal: "कक्षा" } } + e.g. IVथी कक्षा -> tokens { roman { integer: "IV" default_ordinal: "चौथी" key_cardinal: "कक्षा" } } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + roman_graph = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() + roman_numeral_only = pynini.project(roman_graph, "input").optimize() + + devanagari_chars = pynini.project( + pynini.string_file(get_abs_path("data/serial/chars.tsv")), "input" + ).optimize() + + devanagari_word = pynini.closure(devanagari_chars, 1).optimize() + + devanagari_phrase = ( + devanagari_word + pynini.closure((pynini.accep(" ") | pynini.accep("-")) + devanagari_word) + ).optimize() + + separator = (pynini.accep("-") | pynini.accep(" ")).optimize() + + key_before_numeral = ( + pynutil.insert("preserve_order: true ") + + pynutil.insert('key_cardinal: "') + + convert_space(devanagari_phrase) + + pynutil.insert('"') + + pynutil.delete(separator) + + insert_space + + pynutil.insert('integer: "') + + roman_numeral_only + + pynutil.insert('"') + ).optimize() + + numeral_before_key = ( + pynutil.insert("preserve_order: true ") + + pynutil.insert('integer: "') + + roman_numeral_only + + pynutil.insert('"') + + pynutil.delete(separator) + + insert_space + + pynutil.insert('key_cardinal: "') + + convert_space(devanagari_phrase) + + pynutil.insert('"') + ).optimize() + + roman_rows = load_labels(get_abs_path("data/roman/roman_to_spoken.tsv")) + numerals_by_len_desc = sorted((n for n, _ in roman_rows), key=len, reverse=True) + + exception_rows = load_labels(get_abs_path("data/roman/roman_ordinal_exceptions.tsv")) + exception_fused_set = {fused for fused, _ in exception_rows} + + suffix_rows_raw = load_labels(get_abs_path("data/ordinal/suffixes.tsv")) + load_labels( + get_abs_path("data/ordinal/suffixes_map.tsv") + ) + + exception_graphs = [] + for fused, spoken_word in exception_rows: + matched_numeral = next(c for c in numerals_by_len_desc if fused.startswith(c)) + exception_graphs.append( + pynutil.insert('integer: "' + matched_numeral + '"') + + insert_space + + pynutil.insert('default_ordinal: "' + spoken_word + '"') + + pynutil.delete(fused) + ) + glued_ordinal_exceptions_graph = pynini.union(*exception_graphs).optimize() + + regular_row_graphs = [] + for numeral, spoken in roman_rows: + for row in suffix_rows_raw: + + suffix_input = row[0] + suffix_output = row[1] if len(row) > 1 else row[0] + + fused = numeral + suffix_input + if fused in exception_fused_set: + continue + spoken_ordinal = spoken + suffix_output + regular_row_graphs.append( + pynutil.insert('integer: "' + numeral + '"') + + insert_space + + pynutil.insert('default_ordinal: "' + spoken_ordinal + '"') + + pynutil.delete(fused) + ) + glued_ordinal_regular_graph = pynini.union(*regular_row_graphs).optimize() + + roman_glued_ordinal_fields = pynini.union( + pynutil.add_weight(glued_ordinal_exceptions_graph, -0.1), + glued_ordinal_regular_graph, + ).optimize() + + roman_glued_ordinal = ( + pynutil.insert("preserve_order: true ") + + roman_glued_ordinal_fields + + pynini.closure( + pynutil.delete(" ") + + insert_space + + pynutil.insert('key_cardinal: "') + + convert_space(devanagari_phrase) + + pynutil.insert('"'), + 0, + 1, + ) + ).optimize() + + graph = pynini.union(key_before_numeral, numeral_before_key, roman_glued_ordinal).optimize() + + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/serial.py b/nemo_text_processing/text_normalization/hi/taggers/serial.py new file mode 100644 index 000000000..3f2244dd8 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/serial.py @@ -0,0 +1,162 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + + +class SerialFst(GraphFst): + """ + Finite state transducer for classifying serial strings in Hindi. + Handles Devanagari-numeric mixtures, complex delimited number chains, + symbols, and powers. Supports both ASCII (0-9) and Devanagari (०-९) digits. + + e.g. कोविड-19 -> tokens { name: "कोविड-उन्नीस" } + e.g. 5जी -> tokens { name: "पाँच जी" } + e.g. ३जी -> tokens { name: "तीन जी" } + e.g. 2^2 -> tokens { name: "दो स्क्वेर्ड" } + e.g. 2^4 -> tokens { name: "दो टु द पावर चार" } + e.g. 1-800-555 -> tokens { name: "एक-आठ सौ-पाँच सौ पचपन" } + e.g. B-60 -> tokens { name: "बी-साठ" } + e.g. A12 -> tokens { name: "ए बारह" } + e.g. FY2024 -> tokens { name: "एफ वाई दो शून्य दो चार" } + """ + + def __init__( + self, + cardinal: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="serial", kind="classify", deterministic=deterministic) + + digit_graph = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero_graph = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + devanagari_digits = pynini.project( + pynini.union(digit_graph, zero_graph), + "input", + ).optimize() + + any_digit = pynini.union(NEMO_DIGIT, devanagari_digits).optimize() + + # Fetch centralized 1-3 vs 4+ digit logic from cardinal + num_graph = cardinal.code_num_graph + + symbols_graph = pynini.string_file(get_abs_path("data/serial/special_symbols.tsv")).optimize() + + devanagari_chars = pynini.string_file(get_abs_path("data/serial/chars.tsv")).optimize() + + letter_graph = pynini.string_file(get_abs_path("data/address/letters.tsv")) + letter_graph = (letter_graph | pynini.compose(TO_LOWER, letter_graph)).optimize() + latin_letters = letter_graph + pynini.closure(pynutil.insert(" ") + letter_graph) + latin_letters = latin_letters.optimize() + + devanagari_word = pynini.closure(devanagari_chars, 2).optimize() + + delimiter = (pynini.accep("-") | pynini.accep("/") | pynini.accep(" ")).optimize() + + alphas = (latin_letters | devanagari_word).optimize() + segment = (alphas | num_graph | symbols_graph).optimize() + + serial_core = segment + pynini.closure(delimiter + segment, 1) + serial_core = serial_core.optimize() + + serial_graph = serial_core + + all_alphas = pynini.union(NEMO_ALPHA, devanagari_chars).optimize() + + insert_space_alpha_digit = pynini.cdrewrite(pynutil.insert(" "), all_alphas, any_digit, NEMO_SIGMA) + insert_space_digit_alpha = pynini.cdrewrite(pynutil.insert(" "), any_digit, all_alphas, NEMO_SIGMA) + space_inserter = pynini.compose(insert_space_alpha_digit, insert_space_digit_alpha).optimize() + + glued_serial = pynini.compose(space_inserter, serial_core).optimize() + serial_graph = pynini.union(serial_graph, glued_serial).optimize() + + # Reusable mixed alphanumeric-code graph + code_join_char = pynini.union(all_alphas, any_digit, pynini.accep("-"), pynini.accep("/")) + has_letter = pynini.closure(code_join_char) + all_alphas + pynini.closure(code_join_char) + has_digit = pynini.closure(code_join_char) + any_digit + pynini.closure(code_join_char) + mixed_code_only = pynini.intersect( + pynini.intersect(pynini.closure(code_join_char, 1), has_letter), has_digit + ).optimize() + + self.mixed_alphanum_graph = pynini.compose(mixed_code_only, serial_graph).optimize() + + power_special = pynutil.add_weight( + pynini.string_file(get_abs_path("data/serial/power_special.tsv")), -1.0 + ).optimize() + + power_generic = pynutil.add_weight( + (pynutil.delete("^") + pynutil.insert(" टु द पावर ") + num_graph), 1.0 + ).optimize() + + power_suffix = pynini.union(power_special, power_generic).optimize() + power_graph = num_graph + power_suffix + serial_graph = pynini.union(serial_graph, power_graph).optimize() + + serial_graph = pynini.compose(pynini.closure(NEMO_NOT_SPACE, 2), serial_graph).optimize() + + pure_word_slash = pynini.closure(NEMO_ALPHA, 1) + pynini.accep("/") + pynini.closure(NEMO_ALPHA, 1) + + letter_join_char = NEMO_ALPHA | pynini.accep("-") | pynini.accep("/") + contains_latin_letter = pynini.closure(letter_join_char) + NEMO_ALPHA + pynini.closure(letter_join_char) + pure_latin_word = pynini.intersect(pynini.closure(letter_join_char, 1), contains_latin_letter).optimize() + + dimension_pattern = ( + pynini.closure(any_digit, 1) + (pynini.accep("x") | pynini.accep("X")) + pynini.closure(any_digit, 1) + ) + + ordinal_suffixes = pynini.project( + pynini.union( + pynini.string_file(get_abs_path("data/ordinal/suffixes.tsv")), + pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")), + ), + "input", + ).optimize() + ordinal_pattern = pynini.closure(any_digit, 1) + ordinal_suffixes + + date_year_suffix = pynini.project( + pynini.string_file(get_abs_path("data/date/year_suffix.tsv")), + "input", + ).optimize() + date_suffixes = pynini.project( + pynini.string_file(get_abs_path("data/date/suffixes.tsv")), + "input", + ).optimize() + date_pattern = ( + pynini.closure(any_digit, 1) + + pynini.closure(pynini.accep("-") + pynini.closure(any_digit, 1), 0) + + pynini.accep(" ") + + pynini.union(date_year_suffix, date_suffixes) + ) + + exclusions = pure_word_slash | pure_latin_word | dimension_pattern | ordinal_pattern | date_pattern + accepted_inputs = pynini.difference(NEMO_SIGMA, exclusions).optimize() + + serial_graph = pynini.compose(accepted_inputs, serial_graph).optimize() + + self.graph = serial_graph.optimize() + graph = pynutil.insert('name: "') + convert_space(self.graph).optimize() + pynutil.insert('"') + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/telephone.py b/nemo_text_processing/text_normalization/hi/taggers/telephone.py new file mode 100644 index 000000000..d20870c0d --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/telephone.py @@ -0,0 +1,228 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, + NEMO_HI_DIGIT, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + +HI_ZERO_DIGIT = pynini.union("0", "०") +HI_MOBILE_START_DIGITS = pynini.union("६", "७", "८", "९", "6", "7", "8", "9").optimize() +HI_LANDLINE_START_DIGITS = pynini.union("२", "३", "४", "६", "2", "3", "4", "6").optimize() + +delete_zero = pynutil.delete(HI_ZERO_DIGIT) +delete_zero_optional = pynini.closure(delete_zero, 0, 1) +insert_shunya = pynutil.insert('शून्य') + insert_space + +# Load the number mappings from the TSV file +digit_to_word = pynini.string_file(get_abs_path("data/telephone/number.tsv")) +digits = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) +mobile_context = pynini.string_file(get_abs_path("data/telephone/mobile_context.tsv")) +landline_context = pynini.string_file(get_abs_path("data/telephone/landline_context.tsv")) +credit_context = pynini.string_file(get_abs_path("data/telephone/credit_context.tsv")) +pincode_context = pynini.string_file(get_abs_path("data/telephone/pincode_context.tsv")) + +# Reusable optimized graph for any digit token +num_token = pynini.union(digit_to_word, digits, zero).optimize() + + +def generate_mobile(context_keywords: pynini.Fst) -> pynini.Fst: + context_before, context_after = get_context(context_keywords) + + # Filter cardinals to only include allowed digits + mobile_start_digit = pynini.union(HI_MOBILE_START_DIGITS @ digits, HI_MOBILE_START_DIGITS @ digit_to_word) + + country_code_digits = pynini.closure(num_token + insert_space, 1, 3) + country_code = ( + pynutil.insert("country_code: \"") + + context_before + + pynini.cross("+", "प्लस") + + insert_space + + country_code_digits + + pynutil.insert("\" ") + + pynini.closure(delete_space, 0, 1) + ) + + extension_optional = pynini.closure( + pynutil.insert("extension: \"") + + pynini.closure(num_token + insert_space, 1, 3) + + context_after + + pynutil.insert("\" ") + + delete_space, + 0, + 1, + ) + + number_part = mobile_start_digit + insert_space + pynini.closure(num_token + insert_space, 9) + + number_without_country = ( + pynutil.insert("number_part: \"") + + context_before + + delete_zero_optional + + insert_shunya + + number_part + + context_after + + pynutil.insert("\" ") + + delete_space + ) + + number_with_country = ( + country_code + + pynutil.insert("number_part: \"") + + number_part + + context_after + + pynutil.insert("\" ") + + delete_space + ) + + return (pynini.union(number_with_country, number_without_country) + extension_optional).optimize() + + +def get_landline(std_length: int, context_keywords: pynini.Fst) -> pynini.Fst: + context_before, context_after = get_context(context_keywords) + + # Filter cardinals to only include allowed digits + landline_start_digit = pynini.union(HI_LANDLINE_START_DIGITS @ digits, HI_LANDLINE_START_DIGITS @ digit_to_word) + + std_code_graph = ( + delete_zero_optional + insert_shunya + pynini.closure(num_token + insert_space, std_length, std_length) + ) + + landline_digit_count = 9 - std_length + landline_graph = ( + landline_start_digit + + insert_space + + pynini.closure(num_token + insert_space, landline_digit_count, landline_digit_count) + ) + + separator_optional = pynini.closure(pynini.union(pynini.cross("-", ""), pynini.cross(".", "")), 0, 1) + + std_code_in_brackets = ( + delete_zero_optional + + delete_space + + pynutil.delete("(") + + pynini.closure(delete_space, 0, 1) + + std_code_graph + + pynini.closure(delete_space, 0, 1) + + pynutil.delete(")") + ) + + std_part = pynini.union(std_code_graph, std_code_in_brackets) + + return ( + pynutil.insert("number_part: \"") + + context_before + + std_part + + separator_optional + + delete_space + + landline_graph + + context_after + + pynutil.insert("\" ") + ).optimize() + + +def generate_landline(context_keywords: pynini.Fst) -> pynini.Fst: + graph = ( + get_landline(2, context_keywords) + | get_landline(3, context_keywords) + | get_landline(4, context_keywords) + | get_landline(5, context_keywords) + | get_landline(6, context_keywords) + | get_landline(7, context_keywords) + ) + + return graph.optimize() + + +def get_context(keywords: pynini.Fst): + + all_digits = pynini.union(NEMO_HI_DIGIT, NEMO_DIGIT) + + non_digit_char = pynini.difference(NEMO_CHAR, pynini.union(all_digits, NEMO_WHITE_SPACE)) + word = pynini.closure(non_digit_char, 1) + pynini.accep(NEMO_SPACE) + + window = pynini.closure(word, 0, 5) + + before = pynini.closure(keywords + pynini.accep(NEMO_SPACE) + window, 0, 1) + + after = pynini.closure(pynutil.delete(NEMO_SPACE) + window + keywords, 0, 1) + + return before.optimize(), after.optimize() + + +def generate_credit(context_keywords: pynini.Fst) -> pynini.Fst: + context_before, context_after = get_context(context_keywords) + return ( + pynutil.insert("number_part: \"") + + context_before + + pynini.closure(num_token + insert_space, 4) + + context_after + + pynutil.insert("\" ") + + delete_space + ).optimize() + + +def generate_pincode(context_keywords: pynini.Fst) -> pynini.Fst: + context_before, context_after = get_context(context_keywords) + return ( + pynutil.insert("number_part: \"") + + context_before + + pynini.closure(num_token + insert_space, 6) + + context_after + + pynutil.insert("\" ") + + delete_space + ).optimize() + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for tagging telephone numbers, e.g. + ९१५७११४००७ -> telephone { number_part: "शून्य नौ एक पाँच सात एक एक चार शून्य शून्य सात" } + +९१ ९२१०५१५६०६ -> telephone { country_code: "प्लस नौ एक", number_part: "नौ दो एक शून्य पाँच एक पाँच छह शून्य छह" } + १३७४-३०९९८८ -> telephone { number_part: "शून्य एक तीन सात चार तीन शून्य नौ नौ आठ आठ" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization + """ + + def __init__(self): + super().__init__(name="telephone", kind="classify") + + mobile_number = generate_mobile(mobile_context) + landline = generate_landline(landline_context) + credit_card = generate_credit(credit_context) + pincode = generate_pincode(pincode_context) + + graph = ( + pynutil.add_weight(mobile_number, 0.7) + | pynutil.add_weight(landline, 0.8) + | pynutil.add_weight(credit_card, 0.9) + | pynutil.add_weight(pincode, 1) + ) + + self.final = graph.optimize() + self.fst = self.add_tokens(self.final) diff --git a/nemo_text_processing/text_normalization/hi/taggers/time.py b/nemo_text_processing/text_normalization/hi/taggers/time.py new file mode 100644 index 000000000..fc598c2b1 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/time.py @@ -0,0 +1,138 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + HI_DEDH, + HI_DHAI, + HI_PAUNE, + HI_SADHE, + HI_SAVVA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + +# Time patterns specific to time tagger - support both Devanagari and Arabic digits +HI_DOUBLE_ZERO = pynini.union("००", "00") +HI_TIME_FIFTEEN = pynini.union(":१५", ":15") +HI_TIME_THIRTY = pynini.union(":३०", ":30") +HI_TIME_FORTYFIVE = pynini.union(":४५", ":45") + +hours_graph = pynini.string_file(get_abs_path("data/time/hours.tsv")) +minutes_graph = pynini.string_file(get_abs_path("data/time/minutes.tsv")) +seconds_graph = pynini.string_file(get_abs_path("data/time/seconds.tsv")) + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + १२:३०:३० -> time { hours: "बारह" minutes: "तीस" seconds: "तीस" } + १:४० -> time { hours: "एक" minutes: "चालीस" } + १:०० -> time { hours: "एक" } + + Args: + time: GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="time", kind="classify") + + delete_colon = pynutil.delete(":") + cardinal_graph = cardinal.digit | cardinal.teens_and_ties + + self.hours = pynutil.insert("hours: \"") + hours_graph + pynutil.insert("\" ") + self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ") + self.seconds = pynutil.insert("seconds: \"") + seconds_graph + pynutil.insert("\" ") + + # hour minute seconds + graph_hms = ( + self.hours + delete_colon + insert_space + self.minutes + delete_colon + insert_space + self.seconds + ) + + # hour minute + graph_hm = self.hours + delete_colon + insert_space + self.minutes + + # hour + graph_h = self.hours + delete_colon + pynutil.delete(HI_DOUBLE_ZERO) + + # Support all combinations of Devanagari and Arabic digits for dedh/dhai patterns + dedh_dhai_graph = pynini.string_map( + [ + ("१:३०", HI_DEDH), + ("१:30", HI_DEDH), + ("1:३०", HI_DEDH), + ("1:30", HI_DEDH), + ("२:३०", HI_DHAI), + ("२:30", HI_DHAI), + ("2:३०", HI_DHAI), + ("2:30", HI_DHAI), + ] + ) + + savva_numbers = cardinal_graph + pynini.cross(HI_TIME_FIFTEEN, "") + savva_graph = pynutil.insert(HI_SAVVA) + pynutil.insert(NEMO_SPACE) + savva_numbers + + sadhe_numbers = cardinal_graph + pynini.cross(HI_TIME_THIRTY, "") + sadhe_graph = pynutil.insert(HI_SADHE) + pynutil.insert(NEMO_SPACE) + sadhe_numbers + + paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv")) + paune_numbers = paune + pynini.cross(HI_TIME_FORTYFIVE, "") + paune_graph = pynutil.insert(HI_PAUNE) + pynutil.insert(NEMO_SPACE) + paune_numbers + + graph_dedh_dhai = ( + pynutil.insert("morphosyntactic_features: \"") + + dedh_dhai_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + graph_savva = ( + pynutil.insert("morphosyntactic_features: \"") + + savva_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + graph_sadhe = ( + pynutil.insert("morphosyntactic_features: \"") + + sadhe_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + graph_paune = ( + pynutil.insert("morphosyntactic_features: \"") + + paune_graph + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + + final_graph = ( + graph_hms + | pynutil.add_weight(graph_hm, 0.3) + | pynutil.add_weight(graph_h, 0.3) + | pynutil.add_weight(graph_dedh_dhai, 0.1) + | pynutil.add_weight(graph_savva, 0.2) + | pynutil.add_weight(graph_sadhe, 0.2) + | pynutil.add_weight(graph_paune, 0.1) + ) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..04124635e --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py @@ -0,0 +1,181 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.hi.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.hi.taggers.date import DateFst +from nemo_text_processing.text_normalization.hi.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.hi.taggers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.hi.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.hi.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.hi.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.hi.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.hi.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.hi.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.hi.taggers.serial import SerialFst +from nemo_text_processing.text_normalization.hi.taggers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.hi.taggers.time import TimeFst +from nemo_text_processing.text_normalization.hi.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.hi.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"hi_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + fraction_graph = fraction.fst + + date = DateFst(cardinal=cardinal) + date_graph = date.fst + + timefst = TimeFst(cardinal=cardinal) + time_graph = timefst.fst + + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + ordinal_graph = ordinal.fst + + serial = SerialFst(cardinal=cardinal, deterministic=deterministic) + serial_graph = serial.fst + + measure = MeasureFst( + cardinal=cardinal, decimal=decimal, ordinal=ordinal, serial=serial, input_case=input_case + ) + measure_graph = measure.fst + + money = MoneyFst(cardinal=cardinal) + money_graph = money.fst + + whitelist_graph = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ).fst + + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + + word = WordFst(punctuation=punctuation, deterministic=deterministic) + word_graph = word.fst + + roman = RomanFst(deterministic=deterministic) + roman_graph = roman.fst + + telephone = TelephoneFst() + telephone_graph = telephone.fst + + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.05) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(telephone_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(serial_graph, 1.11) + | pynutil.add_weight(roman_graph, 1.1) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct), + ), + 1, + ) + + classify = pynini.union(classify, pynutil.add_weight(word_graph, 100)) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct + pynutil.insert(NEMO_SPACE)), + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph = pynini.union(graph, punct) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/hi/taggers/whitelist.py b/nemo_text_processing/text_normalization/hi/taggers/whitelist.py new file mode 100644 index 000000000..93505f9f7 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/whitelist.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_UPPER, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.hi.utils import ( + augment_labels_with_punct_at_end, + get_abs_path, + load_labels, +) + + +class WhiteListFst(GraphFst): + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file, keep_punct_add_end: bool = False): + whitelist = load_labels(file) + if input_case == INPUT_LOWER_CASED: + whitelist = [[x.lower(), y] for x, y in whitelist] + else: + whitelist = [[x, y] for x, y in whitelist] + + if keep_punct_add_end: + whitelist.extend(augment_labels_with_punct_at_end(whitelist)) + + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/abbreviations.tsv")) + + if deterministic: + graph |= graph.optimize() + else: + graph |= _get_whitelist_graph( + input_case, get_abs_path("data/whitelist/abbreviations.tsv"), keep_punct_add_end=True + ) + + for x in [".", ". "]: + graph |= ( + NEMO_UPPER + + pynini.closure(pynutil.delete(x) + NEMO_UPPER, 2) + + pynini.closure(pynutil.delete("."), 0, 1) + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + self.graph = (convert_space(graph)).optimize() + + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/word.py b/nemo_text_processing/text_normalization/hi/taggers/word.py new file mode 100644 index 000000000..00feb1827 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/taggers/word.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_NOT_SPACE, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.hi.taggers.punctuation import PunctuationFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Hindi words. + e.g. सोना -> tokens { name: "सोना" } + + Args: + punctuation: PunctuationFst + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, punctuation: PunctuationFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + # Define Hindi characters and symbols using pynini.union + HINDI_CHAR = pynini.union( + *[chr(i) for i in range(0x0900, 0x0903 + 1)], # Hindi vowels and consonants + *[chr(i) for i in range(0x0905, 0x0939 + 1)], # More Hindi characters + *[chr(i) for i in range(0x093E, 0x094D + 1)], # Hindi diacritics + ).optimize() + + # Include punctuation in the graph + punct = punctuation.graph + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct.project("input")), 1) + symbols_to_exclude = (pynini.union("$", "€", "₩", "£", "¥", "#", "%") | punct).optimize() + + # Use HINDI_CHAR in the graph + graph = pynini.closure(pynini.difference(HINDI_CHAR, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + # Ensure no spaces around punctuation + graph = pynini.closure(graph + pynini.closure(punct + graph, 0, 1)) + + self.graph = convert_space(graph) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/hi/utils.py b/nemo_text_processing/text_normalization/hi/utils.py new file mode 100644 index 000000000..5d314506e --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/utils.py @@ -0,0 +1,74 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os +import pynini + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path, encoding="utf-8") + labels = list(csv.reader(label_tsv, delimiter="\t")) + label_tsv.close() + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/__init__.py b/nemo_text_processing/text_normalization/hi/verbalizers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/hi/verbalizers/cardinal.py new file mode 100644 index 000000000..9882aa4cf --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/cardinal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> minus twenty three + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign |= pynini.cross("negative: \"true\"", "dash ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + self.numbers = self.optional_sign + integer + delete_tokens = self.delete_tokens(self.numbers) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/date.py b/nemo_text_processing/text_normalization/hi/verbalizers/date.py new file mode 100644 index 000000000..8904f63c8 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/date.py @@ -0,0 +1,74 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date, e.g. + date { day: "एक" month: "अप्रैल" year: "दो हज़ार चौबीस" } -> "एक अप्रैल दो हज़ार चौबीस" + date { month: "अप्रैल" day: "एक" year: "दो हज़ार चौबीस" } -> "अप्रैल एक दो हज़ार चौबीस" + + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self): + super().__init__(name="date", kind="verbalize") + + day = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + month = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + year = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + graph_era = pynutil.delete("era: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + graph_dd_mm = day + NEMO_SPACE + month + + graph_mm_dd = month + NEMO_SPACE + day + + graph_dd_mm_yyyy = day + NEMO_SPACE + month + NEMO_SPACE + year + + graph_mm_dd_yyyy = month + NEMO_SPACE + day + NEMO_SPACE + year + + graph_mm_yyyy = month + NEMO_SPACE + year + + optional_preserve_order = pynini.closure( + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + | pynutil.delete("field_order:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space + ) + + self.graph = ( + (graph_dd_mm | graph_mm_dd | graph_dd_mm_yyyy | graph_mm_dd_yyyy | graph_mm_yyyy | graph_era) + + delete_space + + optional_preserve_order + ) + + final_graph = self.graph + + delete_tokens = self.delete_tokens(final_graph) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/decimal.py b/nemo_text_processing/text_normalization/hi/verbalizers/decimal.py new file mode 100644 index 000000000..ca4636897 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/decimal.py @@ -0,0 +1,53 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import MINUS, NEMO_NOT_QUOTE, GraphFst, insert_space +from nemo_text_processing.text_normalization.hi.taggers.decimal import quantities + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "बारह" fractional_part: "पाँच शून्य शून्य छह" quantity: "अरब" } -> ऋणात्मक बारह दशमलव पाँच शून्य शून्य छह + decimal { integer_part: "बारह" quantity: "billion" } -> बारह अरब + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + delete_space = pynutil.delete(" ") + self.optional_sign = pynini.closure(pynini.cross("negative: \"true\"", MINUS) + delete_space, 0, 1) + self.integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + self.fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + self.fractional = pynutil.insert(" दशमलव ") + self.fractional_default + + self.quantity = ( + delete_space + insert_space + pynutil.delete("quantity: \"") + quantities + pynutil.delete("\"") + ) + self.optional_quantity = pynini.closure(self.quantity, 0, 1) + + graph = self.optional_sign + ( + self.integer + self.quantity | self.integer + delete_space + self.fractional + self.optional_quantity + ) + + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/electronic.py b/nemo_text_processing/text_normalization/hi/verbalizers/electronic.py new file mode 100644 index 000000000..29dcf81d9 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/electronic.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + NEMO_ALPHA, + GraphFst, + capitalized_input_graph, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic addresses. + English words and letters are kept verbatim (Latin script); only digits and + symbols are read out in Hindi. + + Examples: + electronic { username: "kumar" domain: "gmail.com" } -> "kumar एट gmail डॉट com" + electronic { protocol: "https" domain: "google.com/" } -> "https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश google डॉट com फॉरवर्ड स्लैश" + electronic { path: "C:\\Users\\HP\\Desktop" } -> "C कोलन बैकवर्ड स्लैश Users बैकवर्ड स्लैश HP बैकवर्ड स्लैश Desktop" + electronic { domain: "192.168.1.1" } -> "एक नौ दो डॉट एक छह आठ डॉट एक डॉट एक" + electronic { fragment_id: "C₂H₄" } -> "सी दो एच चार" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + symbols_graph = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).optimize() + + ascii_digit_graph = pynini.string_file(get_abs_path("data/telephone/number.tsv")).optimize() + hindi_digit_graph = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize() + hindi_zero_graph = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize() + subscript_digit_graph = pynini.string_file(get_abs_path("data/electronic/subscript_digit.tsv")).optimize() + digit_verbalization = ascii_digit_graph | hindi_digit_graph | hindi_zero_graph | subscript_digit_graph + + protocol_graph = pynini.string_file(get_abs_path("data/electronic/protocols.tsv")).optimize() + + single_digit = digit_verbalization + insert_space + single_symbol = symbols_graph + insert_space + + single_non_alpha = pynutil.add_weight(single_symbol, 1.0) | pynutil.add_weight(single_digit, 1.0) + + # A run of Latin letters is preserved verbatim; digits and symbols verbalize in Hindi. + alpha_run = pynini.closure(NEMO_ALPHA, 1) + insert_space + + # Chemical formulas are spelled out letter-by-letter (element symbols are + # abbreviations, not words), while digits and symbols verbalize in Hindi. + latin_to_hindi_graph = capitalized_input_graph( + pynini.string_file(get_abs_path("data/address/letters.tsv")) + ).optimize() + chem_char = (latin_to_hindi_graph + insert_space) | single_digit | single_symbol + chem_content = pynini.closure(chem_char, 1) + + def make_content(non_alpha_sep=None): + if non_alpha_sep is None: + non_alpha_sep = single_non_alpha + mandatory_sep = pynini.closure(non_alpha_sep, 1) + return ( + pynini.closure(non_alpha_sep, 0) + + pynini.closure(alpha_run + mandatory_sep, 0) + + pynini.closure(alpha_run, 0, 1) + + pynini.closure(non_alpha_sep, 0) + ) + + delete_username_tag = pynutil.delete("username: \"") + delete_domain_tag = pynutil.delete("domain: \"") + delete_protocol_tag = pynutil.delete("protocol: \"") + delete_path_tag = pynutil.delete("path: \"") + delete_fragment_id_tag = pynutil.delete("fragment_id: \"") + delete_quote = pynutil.delete("\"") + + general_content = make_content() + + username_graph = delete_username_tag + general_content + delete_quote + delete_space + pynutil.insert("एट ") + domain_only_graph = delete_domain_tag + general_content + delete_quote + protocol_only_graph = delete_protocol_tag + protocol_graph + insert_space + delete_quote + delete_space + path_graph = delete_path_tag + general_content + delete_quote + + chem_graph = delete_fragment_id_tag + chem_content + delete_quote + + ip_char = single_symbol | single_digit + ip_content = pynini.closure(ip_char, 1) + ip_graph = delete_domain_tag + ip_content + delete_quote + + email_full = username_graph + domain_only_graph + url_full = protocol_only_graph + domain_only_graph + + graph = ( + pynutil.add_weight(url_full, 1.0) + | pynutil.add_weight(email_full, 1.01) + | pynutil.add_weight(path_graph, 1.02) + | pynutil.add_weight(ip_graph, 1.03) + | pynutil.add_weight(domain_only_graph, 1.04) + | pynutil.add_weight(chem_graph, 1.04) + ) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/fraction.py b/nemo_text_processing/text_normalization/hi/verbalizers/fraction.py new file mode 100644 index 000000000..66d944ea7 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/fraction.py @@ -0,0 +1,58 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import MINUS, NEMO_NOT_QUOTE, GraphFst, insert_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. fraction { integer: "तेईस" numerator: "चार" denominator: "छह" }-> तेईस और चार बटा छह + e.g. fraction { numerator: "चार" denominator: "छह" } -> चार बटा छह + + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", MINUS) + pynutil.delete(" "), 0, 1) + + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\" ") + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\" ") + denominator = pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + insert_bata = pynutil.insert(" बटा ") + insert_aur = pynutil.insert(" और ") + graph_quarter = ( + pynutil.delete("morphosyntactic_features: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + fraction_default = numerator + insert_bata + denominator + + self.graph = ( + optional_sign + + pynini.closure(pynini.closure(integer, 0, 1) + insert_space + insert_aur) + + fraction_default + ) | graph_quarter + + graph = self.graph + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py new file mode 100644 index 000000000..cba08057d --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py @@ -0,0 +1,86 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { negative: "true" cardinal { integer: "बारह" } units: "किलोग्राम" } -> ऋणात्मक बारह किलोग्राम + measure { decimal { integer_part: "बारह" fractional_part: "दो" } units: "किलोग्राम" } -> बारह दशमलव दो किलोग्राम + + + Args: + decimal: DecimalFst + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst): + super().__init__(name="measure", kind="verbalize") + + optional_graph_negative = pynini.closure( + pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + insert_space, + 0, + 1, + ) + + unit = ( + pynutil.delete("units: \"") + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.accep("address")) + + pynutil.delete("\"") + + delete_space + ) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_graph_negative + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + optional_graph_negative + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + + graph = (graph_cardinal | graph_decimal) + delete_space + insert_space + unit + + preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + address = ( + pynutil.delete("units: \"address\" ") + + delete_space + + graph_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + + graph |= address + + self.decimal = graph_decimal + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/money.py b/nemo_text_processing/text_normalization/hi/verbalizers/money.py new file mode 100644 index 000000000..1e5da99e4 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/money.py @@ -0,0 +1,102 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.hi.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { currency_maj: "रुपए" integer_part: "बारह" } } -> बारह रुपए + money { currency_maj: "रुपए" integer_part: "बारह" fractional_part: "पचास" currency_min: "पैसे" } -> बारह रुपए पचास पैसे + money { currency_maj: "रुपए" integer_part: "शून्य" fractional_part: "पचास" currency_min: "पैसे" } -> पचास पैसे + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self): + super().__init__(name="money", kind="verbalize") + + sp = pynini.accep(NEMO_SPACE) + + currency_major = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + integer_part = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + fractional_part = ( + pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ) + + currency_minor = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + graph_major_only = integer_part + sp + currency_major + + all_major_names = [maj for maj, _ in load_labels(get_abs_path("data/money/major_minor_currencies.tsv"))] + + major_minor_graphs = [] + minor_only_graphs = [] + + for major in all_major_names: + graph_major_slot = pynutil.delete('currency_maj: "') + pynutil.delete(major) + pynutil.delete('"') + + major_minor_graphs.append( + graph_major_slot + + sp + + integer_part + + pynutil.insert(NEMO_SPACE) + + pynutil.insert(major) + + sp + + fractional_part + + sp + + currency_minor + ) + + minor_only_graphs.append( + graph_major_slot + + sp + + pynutil.delete('integer_part: "शून्य"') + + sp + + fractional_part + + sp + + currency_minor + ) + + graph_major_minor = pynini.union(*major_minor_graphs) + graph_minor_only = pynini.union(*minor_only_graphs) + + decimal_graphs = [] + for major in all_major_names: + decimal_graphs.append( + pynutil.delete('currency_maj: "') + + pynutil.delete(major) + + pynutil.delete('"') + + sp + + integer_part + + sp + + pynutil.insert(" दशमलव ") + + fractional_part + + pynutil.insert(NEMO_SPACE) + + pynutil.insert(major) + ) + graph_decimal_money = pynini.union(*decimal_graphs) + + graph = graph_major_only | graph_major_minor | pynutil.add_weight(graph_minor_only, -0.1) | graph_decimal_money + + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/hi/verbalizers/ordinal.py new file mode 100644 index 000000000..ab88603f6 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/ordinal.py @@ -0,0 +1,38 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing Hindi ordinals, e.g. + ordinal { integer: "दसवां" } -> दसवां + ordinal { integer: "इक्कीसवीं" } -> इक्कीसवीं + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + integer_value = delete_space + pynutil.delete("\"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph = pynutil.delete("integer:") + integer_value + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/hi/verbalizers/post_processing.py new file mode 100644 index 000000000..595180241 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/post_processing.py @@ -0,0 +1,96 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_CHAR, + NEMO_SIGMA, + generator_main, +) +from nemo_text_processing.text_normalization.hi.taggers.punctuation import PunctuationFst +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "hi_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + By default, spaces are removed before punctuation marks like comma, period, etc. + """ + punct_marks_all = PunctuationFst().punct_marks + + # Punctuation marks that should NOT have space before them + # (most punctuation except quotes, dashes, and opening brackets) + quotes = ["'", "\"", "«"] + dashes = ["-", "—"] + brackets = ["<", "{", "(", r"\["] + allow_space_before_punct = quotes + dashes + brackets + + no_space_before_punct = [m for m in punct_marks_all if m not in allow_space_before_punct] + # Add Hindi-specific punctuation + no_space_before_punct.extend(["।", ",", ".", ";", ":", "!", "?"]) + # Remove duplicates + no_space_before_punct = list(set(no_space_before_punct)) + no_space_before_punct = pynini.union(*no_space_before_punct) + + delete_space = pynutil.delete(" ") + + # Delete space before no_space_before_punct marks + non_punct = pynini.difference(NEMO_CHAR, no_space_before_punct).optimize() + graph = ( + pynini.closure(non_punct) + + pynini.closure( + no_space_before_punct | pynutil.add_weight(delete_space + no_space_before_punct, MIN_NEG_WEIGHT) + ) + + pynini.closure(non_punct) + ) + graph = pynini.closure(graph).optimize() + + # Remove space after opening brackets + no_space_after_punct = pynini.union(*brackets) + no_space_after_punct = pynini.cdrewrite(delete_space, no_space_after_punct, NEMO_SIGMA, NEMO_SIGMA).optimize() + graph = pynini.compose(graph, no_space_after_punct).optimize() + + return graph diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py new file mode 100644 index 000000000..c28084a77 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py @@ -0,0 +1,98 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_zero_or_one_space, + insert_space, +) +from nemo_text_processing.text_normalization.hi.utils import get_abs_path + + +class RomanFst(GraphFst): + """ + Finite state transducer for verbalizing Roman numerals in Hindi. + roman { preserve_order: true key_cardinal: "भास्कर" integer: "II" } -> भास्कर दो + roman { preserve_order: true key_cardinal: "कक्षा" integer: "XII" } -> कक्षा बारह + roman { preserve_order: true integer: "XII" default_ordinal: "बारहवीं" key_cardinal: "कक्षा" } -> बारहवीं कक्षा + roman { preserve_order: true integer: "IV" default_ordinal: "चौथी" key_cardinal: "कक्षा" } -> चौथी कक्षा + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + + roman_to_spoken = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() + + key_cardinal = ( + pynutil.delete('key_cardinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ).optimize() + + integer = (pynutil.delete('integer: "') + roman_to_spoken + pynutil.delete('"')).optimize() + + default_ordinal = ( + pynutil.delete('default_ordinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ).optimize() + + ignore_integer = ( + pynutil.delete('integer: "') + pynutil.delete(pynini.closure(NEMO_NOT_QUOTE, 1)) + pynutil.delete('"') + ).optimize() + + drop_preserve_order = pynini.closure( + delete_zero_or_one_space + + pynutil.delete("preserve_order:") + + delete_zero_or_one_space + + pynutil.delete("true") + + delete_zero_or_one_space, + 0, + 1, + ).optimize() + + key_first = ( + drop_preserve_order + + key_cardinal + + delete_zero_or_one_space + + insert_space + + integer + + drop_preserve_order + ).optimize() + + numeral_first = ( + drop_preserve_order + + integer + + delete_zero_or_one_space + + insert_space + + key_cardinal + + drop_preserve_order + ).optimize() + + glued_ordinal = ( + drop_preserve_order + + ignore_integer + + delete_zero_or_one_space + + default_ordinal + + pynini.closure(delete_zero_or_one_space + insert_space + key_cardinal, 0, 1) + + drop_preserve_order + ).optimize() + + graph = pynini.union(key_first, numeral_first, glued_ordinal).optimize() + + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/telephone.py b/nemo_text_processing/text_normalization/hi/verbalizers/telephone.py new file mode 100644 index 000000000..55ebeab01 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/telephone.py @@ -0,0 +1,72 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone numbers, e.g. + telephone { country_code: "प्लस नौ एक", number_part: "नौ दो एक शून्य पाँच एक पाँच छह शून्य छह" } -> प्लस नौ एक नौ दो एक शून्य पाँच एक पाँच छह शून्य छह + telephone { number_part: "शून्य एक तीन सात चार तीन शून्य नौ नौ आठ आठ" } -> शून्य एक तीन सात चार तीन शून्य नौ नौ आठ आठ + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + optional_country_code = pynini.closure( + pynutil.delete("country_code: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + insert_space, + 0, + 1, + ) + + number_part = ( + pynutil.delete("number_part: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(NEMO_SPACE), MIN_NEG_WEIGHT), 0, 1) + + pynutil.delete("\"") + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynutil.delete("extension: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\""), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/time.py b/nemo_text_processing/text_normalization/hi/verbalizers/time.py new file mode 100644 index 000000000..df232e3cd --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/time.py @@ -0,0 +1,81 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + time { hours: "बारह" minutes: "दस" seconds: "दस" } -> बारह बजकर दस मिनट दस सेकंड + time { hours: "सात" minutes: "चालीस"" } -> सात बजकर चालीस मिनट + time { hours: "दस" } -> दस बजे + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst): + super().__init__(name="time", kind="verbalize") + + hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + insert_space + + minute = ( + pynutil.delete("minutes: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + insert_space + ) + + second = ( + pynutil.delete("seconds: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + insert_space + ) + + insert_minute = pynutil.insert("मिनट") + insert_second = pynutil.insert("सेकंड") + insert_bajkar = pynutil.insert("बजकर") + insert_baje = pynutil.insert("बजे") + + # hour minute second + graph_hms = ( + hour + + delete_space + + insert_bajkar + + insert_space + + minute + + delete_space + + insert_minute + + insert_space + + second + + delete_space + + insert_second + ) + + graph_quarter = ( + pynutil.delete("morphosyntactic_features: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + # hour minute + graph_hm = hour + delete_space + insert_bajkar + insert_space + minute + delete_space + insert_minute + + # hour + graph_h = hour + delete_space + insert_baje + + self.graph = graph_hms | graph_hm | graph_h | graph_quarter + + final_graph = self.graph + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/hi/verbalizers/verbalize.py new file mode 100644 index 000000000..bd6ca4b5b --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/verbalize.py @@ -0,0 +1,94 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst +from nemo_text_processing.text_normalization.hi.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.hi.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.hi.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.hi.verbalizers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.hi.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.hi.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.hi.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.hi.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.hi.verbalizers.roman import RomanFst +from nemo_text_processing.text_normalization.hi.verbalizers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.hi.verbalizers.time import TimeFst +from nemo_text_processing.text_normalization.hi.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + decimal = DecimalFst(deterministic=deterministic) + decimal_graph = decimal.fst + + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + fraction_graph = fraction.fst + + date = DateFst() + date_graph = date.fst + + time = TimeFst(cardinal=cardinal) + time_graph = time.fst + + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + + measure = MeasureFst(cardinal=cardinal, decimal=decimal) + measure_graph = measure.fst + + money = MoneyFst() + money_graph = money.fst + + telephone = TelephoneFst() + telephone_graph = telephone.fst + + electronic = ElectronicFst(deterministic=deterministic) + electronic_graph = electronic.fst + + roman = RomanFst(deterministic=deterministic) + roman_graph = roman.fst + + whitelist_graph = WhiteListFst(deterministic=deterministic).fst + + graph = ( + cardinal_graph + | decimal_graph + | fraction_graph + | date_graph + | time_graph + | measure_graph + | money_graph + | ordinal_graph + | whitelist_graph + | telephone_graph + | electronic_graph + | roman_graph + ) + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/hi/verbalizers/verbalize_final.py new file mode 100644 index 000000000..948c820b9 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/verbalize_final.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.hi.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.hi.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"en_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + if deterministic: + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + else: + graph = delete_space + types + delete_space + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/hi/verbalizers/whitelist.py new file mode 100644 index 000000000..d846dfa58 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/whitelist.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "misses" } } -> misses + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/word.py b/nemo_text_processing/text_normalization/hi/verbalizers/word.py new file mode 100644 index 000000000..30a6d4478 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/verbalizers/word.py @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing Hindi words. + e.g. tokens { name: "सोना" } -> सोना + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + punct = pynini.union("!", "?", ".", ",", "-", ":", ";", "।") # Add other punctuation marks as needed + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + + # Ensure no spaces around punctuation + graph = char + pynini.closure(delete_space + punct, 0, 1) + + # Explicitly remove spaces before punctuation + remove_space_before_punct = pynini.cdrewrite(pynini.cross(" ", ""), "", punct, NEMO_SIGMA) + graph = graph @ remove_space_before_punct + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hu/graph_utils.py b/nemo_text_processing/text_normalization/hu/graph_utils.py index 971b8bdd6..be7c157fd 100644 --- a/nemo_text_processing/text_normalization/hu/graph_utils.py +++ b/nemo_text_processing/text_normalization/hu/graph_utils.py @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space from pynini.lib import byte +from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space + _ALPHA_UPPER = "AÁBCDEÉFGHIÍJKLMNOÓÖŐPQRSTUÚÜŰVWXYZ" _ALPHA_LOWER = "aábcdeéfghiíjklmnoóöőpqrstuúüűvwxyz" _VOWELS = "AÁEÉIÍOÓÖŐUÚÜŰaáeéiíoóöőuúüű" diff --git a/nemo_text_processing/text_normalization/hu/taggers/cardinal.py b/nemo_text_processing/text_normalization/hu/taggers/cardinal.py index 91efa3085..c9c5c3063 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/hu/taggers/cardinal.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.hu.graph_utils import HU_ALPHA from nemo_text_processing.text_normalization.hu.utils import get_abs_path -from pynini.lib import pynutil def make_million(word: str, hundreds: 'pynini.FstLike', deterministic=False): @@ -61,7 +62,7 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': fst: A pynini.FstLike object """ cardinal_separator = pynini.string_map([".", NEMO_SPACE]) - exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string up_to_three_digits = up_to_three_digits - "000" - "00" - "0" @@ -245,7 +246,7 @@ def __init__(self, deterministic: bool = True): self.graph = ( ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ clean_output ) @@ -256,12 +257,12 @@ def __init__(self, deterministic: bool = True): zero_space + digit, ).optimize() self.three_digits_read = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit, - zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit, + zero_space + ((NEMO_DIGIT**2) @ graph_tens), zero_space + zero_space + digit, ).optimize() self.four_digits_read = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 3)) @ self.graph, zero_space + self.three_digits_read + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**3)) @ self.graph, zero_space + self.three_digits_read ).optimize() self.graph |= graph_zero diff --git a/nemo_text_processing/text_normalization/hu/taggers/date.py b/nemo_text_processing/text_normalization/hu/taggers/date.py index efa5e44c4..da410dc31 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/date.py +++ b/nemo_text_processing/text_normalization/hu/taggers/date.py @@ -14,10 +14,11 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_DIGIT, NEMO_SPACE, GraphFst from nemo_text_processing.text_normalization.hu.graph_utils import TO_LOWER, TO_UPPER from nemo_text_processing.text_normalization.hu.utils import get_abs_path, load_labels -from pynini.lib import pynutil def get_suffixed_days(labels): @@ -40,7 +41,7 @@ def day_inflector(number, day): Args: number: the day number day: the day name - + Returns: a list of expanded forms, two per ending. """ @@ -70,7 +71,7 @@ def day_adj_endings(number, word, basic=True): 1-jei -> elsejei 2-i -> másodiki 2-ai -> másodikai - 4-i -> negyediki + 4-i -> negyediki 4-ei -> negyedikei This is based on other -i adjectives, because these forms are rare. """ @@ -102,7 +103,7 @@ def day_adj_endings(number, word, basic=True): class DateFst(GraphFst): """ - Finite state transducer for classifying date, e.g. + Finite state transducer for classifying date, e.g. "2010. április 1." -> date { year: "kettőezer-tíz" month: "április" day: "elseje" preserve_order: true } "2010. ápr. 1." -> date { year: "kettőezer-tíz" month: "április" day: "elseje" preserve_order: true } "2010. IV. 1." -> date { year: "kettőezer-tíz" month: "április" day: "elseje" preserve_order: true } diff --git a/nemo_text_processing/text_normalization/hu/taggers/decimal.py b/nemo_text_processing/text_normalization/hu/taggers/decimal.py index 9e30b55a3..10ae4a8fe 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/hu/taggers/decimal.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SIGMA, GraphFst, insert_space from nemo_text_processing.text_normalization.hu.utils import get_abs_path, load_labels, naive_inflector -from pynini.lib import pynutil quantities = load_labels(get_abs_path("data/number/quantities.tsv")) @@ -45,7 +46,7 @@ def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstL e.g. 1 millió -> integer_part: "egy" quantity: "millió" e.g. 1,4 million -> integer_part: "egy" fractional_part: "négy" quantity: "millió" - Args: + Args: decimal: decimal FST cardinal_up_to_hundred: cardinal FST """ @@ -67,7 +68,7 @@ def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_hundred: 'pynini.FstL class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. + Finite state transducer for classifying decimal, e.g. -11,4006 milliárd -> decimal { negative: "true" integer_part: "tizenegy" fractional_part: "négyezer-hat tízezred" quantity: "milliárd" preserve_order: true } 1 milliárd -> decimal { integer_part: "egy" quantity: "milliárd" preserve_order: true } Args: @@ -100,7 +101,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ]: for modifier in ["", "tíz", "száz"]: decimal_number |= ( - (NEMO_DIGIT ** order + (NEMO_DIGIT - "0")) + (NEMO_DIGIT**order + (NEMO_DIGIT - "0")) @ pynini.cdrewrite(pynini.cross("0", ""), "[BOS]", "", NEMO_SIGMA) @ cardinal_graph + final_zero diff --git a/nemo_text_processing/text_normalization/hu/taggers/electronic.py b/nemo_text_processing/text_normalization/hu/taggers/electronic.py index f7d1fe4d5..67160e99e 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/hu/taggers/electronic.py @@ -11,13 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space -from nemo_text_processing.text_normalization.es.utils import get_abs_path, load_labels from pynini.lib import pynutil -common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] -symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] +from nemo_text_processing.text_normalization.de.utils import get_abs_path, load_labels +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + at, + colon, + domain_string, + double_quotes, + double_slash, + http, + https, + period, + protocol_string, + username_string, + www, +) class ElectronicFst(GraphFst): @@ -34,45 +50,65 @@ class ElectronicFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="electronic", kind="classify", deterministic=deterministic) - dot = pynini.accep(".") - accepted_common_domains = pynini.union(*common_domains) - accepted_symbols = pynini.union(*symbols) - dot - accepted_characters = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols) - acceepted_characters_with_dot = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols | dot) + period_fst = pynini.accep(period) + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + # all symbols + symbols_no_period = pynini.difference(symbols, period_fst) # alphabet of accepted symbols excluding the '.' + accepted_characters = pynini.closure( + (NEMO_ALPHA | NEMO_DIGIT | symbols_no_period), 1 + ) # alphabet of accepted chars excluding the '.' + all_characters = pynini.closure( + (NEMO_ALPHA | NEMO_DIGIT | symbols), 1 + ) # alphabet of accepted chars including the '.' + + # domains + domain = period_fst + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters + pynini.closure(domain, 1)) + + pynutil.insert(double_quotes) + ) # email username = ( - pynutil.insert("username: \"") - + acceepted_characters_with_dot - + pynutil.insert("\"") - + pynini.cross('@', ' ') + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + all_characters + + pynutil.insert(double_quotes) + + pynini.cross(at, NEMO_SPACE) ) - domain_graph = accepted_characters + dot + accepted_characters - domain_graph = pynutil.insert("domain: \"") + domain_graph + pynutil.insert("\"") - domain_common_graph = ( - pynutil.insert("domain: \"") - + accepted_characters - + accepted_common_domains - + pynini.closure((accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1) - + pynutil.insert("\"") + email = username + domain_graph + + # social media tags + tag = ( + pynutil.delete(at) + + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + pynutil.insert(double_quotes) ) - graph = (username + domain_graph) | domain_common_graph # url - protocol_start = pynini.accep("https://") | pynini.accep("http://") + protocol_start = pynini.accep(https + colon + double_slash) | pynini.accep(http + colon + double_slash) protocol_end = ( - pynini.accep("www.") + pynini.accep(www + period) if deterministic else ( - pynini.accep("www.") - | pynini.cross("www.", "vé vé vé.") - | pynini.cross("www.", "dupla vé dupla vé dupla vé.") - | pynini.cross("www.", "kettős vé kettős vé kettős vé.") + pynini.accep(www + period) + | pynini.cross(www + period, "vé vé vé.") + | pynini.cross(www + period, "dupla vé dupla vé dupla vé.") + | pynini.cross(www + period, "kettős vé kettős vé kettős vé.") ) ) protocol = protocol_start | protocol_end | (protocol_start + protocol_end) - protocol = pynutil.insert("protocol: \"") + protocol + pynutil.insert("\"") - graph |= protocol + insert_space + (domain_graph | domain_common_graph) + protocol = ( + pynutil.insert(protocol_string + colon + NEMO_SPACE + double_quotes) + + protocol + + pynutil.insert(double_quotes) + ) + url = protocol + pynutil.insert(NEMO_SPACE) + (domain_graph) + + graph = url | domain_graph | email | tag self.graph = graph final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) diff --git a/nemo_text_processing/text_normalization/hu/taggers/fraction.py b/nemo_text_processing/text_normalization/hu/taggers/fraction.py index 53f630cf2..e67146bc5 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/hu/taggers/fraction.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.hu.utils import get_abs_path -from pynini.lib import pynutil fraction_symbols = pynini.string_file(get_abs_path("data/fractions/fraction_symbols.tsv")) diff --git a/nemo_text_processing/text_normalization/hu/taggers/measure.py b/nemo_text_processing/text_normalization/hu/taggers/measure.py index 8f93d1758..f2c3a2368 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hu/taggers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_NON_BREAKING_SPACE, @@ -23,7 +25,6 @@ ) from nemo_text_processing.text_normalization.hu.taggers.whitelist import load_inflected from nemo_text_processing.text_normalization.hu.utils import get_abs_path -from pynini.lib import pynutil unit_singular = pynini.string_file(get_abs_path("data/measures/measurements.tsv")) @@ -60,7 +61,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) optional_unit_denominator = pynini.closure( - pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, 0, 1, + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, ) unit_singular_graph = ( diff --git a/nemo_text_processing/text_normalization/hu/taggers/money.py b/nemo_text_processing/text_normalization/hu/taggers/money.py index 5cfafdba1..f45e60835 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/money.py +++ b/nemo_text_processing/text_normalization/hu/taggers/money.py @@ -15,6 +15,8 @@ import re import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -30,7 +32,6 @@ load_labels, naive_inflector, ) -from pynini.lib import pynutil min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) maj_singular = pynini.string_file((get_abs_path("data/money/currency.tsv"))) diff --git a/nemo_text_processing/text_normalization/hu/taggers/ordinal.py b/nemo_text_processing/text_normalization/hu/taggers/ordinal.py index 4319ee372..a63a9f02a 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/hu/taggers/ordinal.py @@ -16,15 +16,16 @@ # Russian minimally supervised number grammar. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.hu.taggers.cardinal import filter_punctuation from nemo_text_processing.text_normalization.hu.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. "2." -> ordinal { integer: "második" } } Args: diff --git a/nemo_text_processing/text_normalization/hu/taggers/telephone.py b/nemo_text_processing/text_normalization/hu/taggers/telephone.py index 95ea04643..b031ca5dc 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/hu/taggers/telephone.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_SPACE, GraphFst, @@ -22,7 +24,6 @@ ) from nemo_text_processing.text_normalization.hu.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.hu.utils import get_abs_path -from pynini.lib import pynutil class TelephoneFst(GraphFst): @@ -40,8 +41,8 @@ class TelephoneFst(GraphFst): https://en.wikipedia.org/wiki/Telephone_numbers_in_Hungary Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) """ def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/hu/taggers/time.py b/nemo_text_processing/text_normalization/hu/taggers/time.py index 138ad97a0..43e067fef 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/time.py +++ b/nemo_text_processing/text_normalization/hu/taggers/time.py @@ -15,6 +15,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SPACE, @@ -28,7 +30,6 @@ load_labels, naive_inflector, ) -from pynini.lib import pynutil QUARTERS = {15: "negyed", 30: "fél", 45: "háromnegyed"} @@ -70,7 +71,7 @@ class TimeFst(GraphFst): "09:00 óra" -> time { hours: "2" } "02:15:10 óra" -> time { hours: "2" minutes: "15" seconds: "10"} "negyed 2" -> time { minutes: "15" hours: "1" } - + Args: deterministic: if True will provide a single transduction option, for False multiple transduction are generated (used for audio-based normalization) @@ -179,7 +180,11 @@ def hours_to_pairs(): final_time_zone = ( pynini.accep(" ") + pynutil.insert("zone: \"") + convert_space(time_zone_graph) + pynutil.insert("\"") ) - final_time_zone_optional = pynini.closure(final_time_zone, 0, 1,) + final_time_zone_optional = pynini.closure( + final_time_zone, + 0, + 1, + ) # This might be better as just the inflected forms hour_only_delimited = ( diff --git a/nemo_text_processing/text_normalization/hu/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/hu/taggers/tokenize_and_classify.py index 6ad09bce1..8c269bb00 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/hu/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_WHITE_SPACE, GraphFst, @@ -36,7 +37,7 @@ from nemo_text_processing.text_normalization.hu.taggers.time import TimeFst from nemo_text_processing.text_normalization.hu.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.hu.taggers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -68,13 +69,14 @@ def __init__( os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" far_file = os.path.join( - cache_dir, f"_{input_case}_hu_tn_{deterministic}_deterministic{whitelist_file}.far" + cache_dir, + f"_{input_case}_hu_tn_{deterministic}_deterministic{whitelist_file}.far", ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") self.cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = self.cardinal.fst @@ -85,10 +87,17 @@ def __init__( self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic) decimal_graph = self.decimal.fst - self.fraction = FractionFst(cardinal=self.cardinal, ordinal=self.ordinal, deterministic=deterministic) + self.fraction = FractionFst( + cardinal=self.cardinal, + ordinal=self.ordinal, + deterministic=deterministic, + ) fraction_graph = self.fraction.fst self.measure = MeasureFst( - cardinal=self.cardinal, decimal=self.decimal, fraction=self.fraction, deterministic=deterministic + cardinal=self.cardinal, + decimal=self.decimal, + fraction=self.fraction, + deterministic=deterministic, ) measure_graph = self.measure.fst self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic) @@ -100,7 +109,11 @@ def __init__( telephone_graph = self.telephone.fst self.electronic = ElectronicFst(deterministic=deterministic) electronic_graph = self.electronic.fst - self.money = MoneyFst(cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic) + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) money_graph = self.money.fst self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) whitelist_graph = self.whitelist.fst @@ -117,7 +130,7 @@ def __init__( | pynutil.add_weight(decimal_graph, 1.1) | pynutil.add_weight(money_graph, 1.1) | pynutil.add_weight(telephone_graph, 1.1) - | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(electronic_graph, 1.11) | pynutil.add_weight(word_graph, 200) ) punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") @@ -146,4 +159,4 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") + logger.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/hu/taggers/whitelist.py b/nemo_text_processing/text_normalization/hu/taggers/whitelist.py index 95cbfee22..88e7d669f 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/hu/taggers/whitelist.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.hu.utils import get_abs_path, load_labels, naive_inflector -from pynini.lib import pynutil def load_inflected(filename, input_case, singular_only=False, skip_spaces=True): diff --git a/nemo_text_processing/text_normalization/hu/taggers/word.py b/nemo_text_processing/text_normalization/hu/taggers/word.py index 1d7a6e9b0..31a71ef14 100644 --- a/nemo_text_processing/text_normalization/hu/taggers/word.py +++ b/nemo_text_processing/text_normalization/hu/taggers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/hu/utils.py b/nemo_text_processing/text_normalization/hu/utils.py index 8a87a3166..a5fb4fc3c 100644 --- a/nemo_text_processing/text_normalization/hu/utils.py +++ b/nemo_text_processing/text_normalization/hu/utils.py @@ -49,7 +49,7 @@ def load_inflection(abs_path): Args: abs_path: absolute path - + Returns dictionary of mappings of word endings to lists of case endings. """ @@ -97,7 +97,7 @@ def inflect_abbreviation(abbr: str, word: str, singular_only=False): word: the base (nominative singular) form of the expansion of abbr singular_only: whether or not to add plural forms - + Returns a list of tuples containing the inflected abbreviation and its expansion. """ @@ -133,7 +133,7 @@ def naive_inflector(abbr: str, word: str, singular_only=False): word: the base (nominative singular) form of the expansion of abbr singular_only: whether or not to add plural forms - + Returns a list of tuples containing the inflected abbreviation and its expansion. """ diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/hu/verbalizers/cardinal.py index 3e80e3540..9bed76d9e 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/date.py b/nemo_text_processing/text_normalization/hu/verbalizers/date.py index bda801b34..047c736b9 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order + class DateFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/decimal.py b/nemo_text_processing/text_normalization/hu/verbalizers/decimal.py index 8e5d8c880..93070b050 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/electronic.py b/nemo_text_processing/text_normalization/hu/verbalizers/electronic.py index 491b94dfe..3847970ee 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/electronic.py @@ -12,15 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, + NEMO_SPACE, GraphFst, + at, + colon, delete_preserve_order, - insert_space, + domain_string, + double_quotes, + double_slash, + http, + https, + period, + protocol_string, + username_string, + www, ) from nemo_text_processing.text_normalization.hu.utils import get_abs_path -from pynini.lib import pynutil digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/number/digit.tsv"))) zero = pynini.invert(pynini.string_file(get_abs_path("data/number/zero.tsv"))) @@ -29,6 +41,12 @@ server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) +accept_space = pynini.accep(NEMO_SPACE) +delete_username = pynutil.delete(username_string + colon + NEMO_SPACE + double_quotes) +delete_double_quotes = pynutil.delete(double_quotes) +delete_domain = pynutil.delete(domain_string + colon + NEMO_SPACE + double_quotes) +delete_protocol = pynutil.delete(protocol_string + colon + NEMO_SPACE + double_quotes) + class ElectronicFst(GraphFst): """ @@ -48,36 +66,41 @@ def __init__(self, deterministic: bool = True): graph_digit = digit_no_zero | zero def add_space_after_char(): - return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + ( - NEMO_NOT_QUOTE - pynini.accep(" ") + return pynini.closure(NEMO_NOT_QUOTE - accept_space + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - accept_space ) + hungarian_at = [ + "kukacjel ", + "csiga ", + "ormány ", + "farkas á ", + "bejgli ", + "at-jel ", + ] at_sign = pynutil.insert("kukac ") if not deterministic: - at_sign |= pynutil.insert("kukacjel ") - at_sign |= pynutil.insert("csiga ") - at_sign |= pynutil.insert("ormány ") - at_sign |= pynutil.insert("farkas á ") - at_sign |= pynutil.insert("bejgli ") - at_sign |= pynutil.insert("at-jel ") + for sign in hungarian_at: + at_sign |= pynutil.insert(sign) verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) - user_name = pynutil.delete("username: \"") + add_space_after_char() + pynutil.delete("\"") + user_name = delete_username + add_space_after_char() + delete_double_quotes user_name @= verbalize_characters convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common - domain = convert_defaults + pynini.closure(insert_space + convert_defaults) + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) domain @= verbalize_characters - domain = pynutil.delete("domain: \"") + domain + pynutil.delete("\"") + domain = delete_domain + domain + delete_double_quotes protocol = ( - pynutil.delete("protocol: \"") + delete_protocol + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) - + pynutil.delete("\"") + + delete_double_quotes ) - self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | ( - user_name + pynini.accep(" ") + at_sign + domain + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + at_sign + domain | (at_sign + user_name) ) delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/fraction.py b/nemo_text_processing/text_normalization/hu/verbalizers/fraction.py index 2f3019b6e..8b477a5fe 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/fraction.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_preserve_order, insert_space, ) -from pynini.lib import pynutil class FractionFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/measure.py b/nemo_text_processing/text_normalization/hu/verbalizers/measure.py index b9bcd190e..675659044 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/measure.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_preserve_order, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): @@ -27,7 +28,7 @@ class MeasureFst(GraphFst): Finite state transducer for verbalizing measure, e.g. measure { cardinal { integer: "zwei" units: "unzen" } } -> "zwei unzen" measure { cardinal { integer_part: "zwei" quantity: "millionen" units: "unzen" } } -> "zwei millionen unzen" - + Args: decimal: decimal GraphFst cardinal: cardinal GraphFst diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/money.py b/nemo_text_processing/text_normalization/hu/verbalizers/money.py index 342abfd09..316fcc7a0 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_preserve_order + class MoneyFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/hu/verbalizers/ordinal.py index 53b0cdf41..eb5369931 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/ordinal.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/telephone.py b/nemo_text_processing/text_normalization/hu/verbalizers/telephone.py index 778407bed..b52e6efb7 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/telephone.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/telephone.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + class TelephoneFst(GraphFst): """ @@ -33,7 +34,11 @@ def __init__(self, deterministic: bool = True): country_code = pynutil.delete("country_code: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") - optional_country_code = pynini.closure(country_code + delete_space + insert_space, 0, 1,) + optional_country_code = pynini.closure( + country_code + delete_space + insert_space, + 0, + 1, + ) number_part = ( pynutil.delete("number_part: \"") @@ -52,6 +57,8 @@ def __init__(self, deterministic: bool = True): 1, ) - graph = pynini.union(optional_country_code + number_part + optional_extension,) + graph = pynini.union( + optional_country_code + number_part + optional_extension, + ) delete_tokens = self.delete_tokens(graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/time.py b/nemo_text_processing/text_normalization/hu/verbalizers/time.py index a0efd5a4e..9f960b100 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -23,7 +25,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/hu/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/hu/verbalizers/verbalize_final.py index 27fcafae1..86db540d0 100644 --- a/nemo_text_processing/text_normalization/hu/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/hu/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -24,7 +25,7 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.hu.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): @@ -47,7 +48,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"hu_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst @@ -67,4 +68,4 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") + logger.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/hy/__init__.py b/nemo_text_processing/text_normalization/hy/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hy/data/__init__.py b/nemo_text_processing/text_normalization/hy/data/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hy/data/currency.tsv b/nemo_text_processing/text_normalization/hy/data/currency.tsv new file mode 100644 index 000000000..a1d8e03dd --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/currency.tsv @@ -0,0 +1,52 @@ +aed դիրհամ +ARS արգենտինական պեսո +֏ դրամ +֏ հայկական դրամ +a$ ավստրալական դոլար +br բելառուսական ռուբլի +₿ բիթքոին +£ ֆունտ +£ բրիտանական ֆունտ +ca$ կանադական դոլար +元 չինական յեն +元 յեն +kr դանիական կրոն +$ դոլար +€ եվրո +₾ վրացական լարի +₾ լարի +₹ հնդկական ռուփի +₹ ռուփի +﷼ պարսկական ռիալ +﷼ ռիալ +₪ իսրայելական շեկել +₪ շեկել +¥ ճապոնական յեն +¥ յեն +₸ ղազախական տենգե +₸ տենգե +som ղրղզական սոմ +som սոմ +ل.ل լիբանանյան ֆունտ +ل.ل լիբանանյան լիրա +nz$ նորզելանդական դոլլր +kr նորվեգական կրոն +zł լեհական զլոտի +zł զլոտի +£ ֆունտ ստերլինգ +£ ֆունտ +₽ ռուսական ռուբլի +₽ ռուբլի +rsd սերբական դինար +s$ սինգապուրի դոլար +₩ կորեական վոն +kr շվեդական կրոն +chf շվեյցարական ֆրանկ +£s սիրիական ֆունտ +₺ թուրքական լիրա +₴ ուկրաինական գրիվնա +$ ամերիկյան դոլար +$ ամն դոլար +som ուզբեկական սոմ +₩ վոն +¥ յեն \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/measurement_dates.tsv b/nemo_text_processing/text_normalization/hy/data/measurement_dates.tsv new file mode 100644 index 000000000..00ac92f9d --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/measurement_dates.tsv @@ -0,0 +1,8 @@ +թ. թվական +թթ. թվականներ +դ. դար +դդ. դարեր +թ․ թվական +թթ․ թվականներ +դ․ դար +դդ․ դարեր \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/measurements.tsv b/nemo_text_processing/text_normalization/hy/data/measurements.tsv new file mode 100644 index 000000000..37fe39154 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/measurements.tsv @@ -0,0 +1,44 @@ +°F ֆարենհայթ +°C ցելսիուս +K կելվին +կմ կիլոմետր +կմ/ժ կիլոմետր ժամ +մ մետր +մ/ժ մետր ժամ +սմ սանտիմետր +մմ միլիմետր +հա հեկտար +մղն մղոն +մ² քառակուսի մետր +կմ² քառակուսի կիլոմետր +% տոկոս +Հց հերց +կՎտ կիլովատտ +կՎտ կիլո վատտ +կՎ/Ժ կիլովատտ ժամ +Վտ/ժ վատտ ժամ +Վտ վատտ +ձ.ու. ձիաուժ +ձ․ու․ ձիաուժ +մգ միլիգրամ +կգ կիլոգրամ +Վ վոլտ +ժ ժամ +վ վայրկյան +ր րոպե +մ³ խորանարդ մետր +գ գրամ +տ տոննա +կբ կիլոբայթ +մբ մեգաբայթ +գբ գիգաբայթ +գբ գեգաբայթ +տբ տերաբայթ +տբ տեռաբայթ +կՎ կիլո վոլտ +մՎ մեգա վոլտ +Ա ամպեր +մԱ միլի ամպեր +մվ միլի վայրկյան +դմ դեցիմետր +սմ² քառակուսի սանտիմետր diff --git a/nemo_text_processing/text_normalization/hy/data/numbers/__init__.py b/nemo_text_processing/text_normalization/hy/data/numbers/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hy/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/hy/data/numbers/digit.tsv new file mode 100644 index 000000000..42feb82d5 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +մեկ 1 +երկու 2 +երեք 3 +չորս 4 +հինգ 5 +վեց 6 +յոթ 7 +ութ 8 +ինը 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/numbers/quantities.tsv b/nemo_text_processing/text_normalization/hy/data/numbers/quantities.tsv new file mode 100644 index 000000000..8b53443af --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/numbers/quantities.tsv @@ -0,0 +1,3 @@ +միլիոն +միլիարդ +տրիլիոն \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/numbers/ties.tsv b/nemo_text_processing/text_normalization/hy/data/numbers/ties.tsv new file mode 100644 index 000000000..0faae0a9f --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +քսան 2 +երեսուն 3 +քառասուն 4 +հիսուն 5 +վաթսուն 6 +յոթանասուն 7 +ութսուն 8 +իննսուն 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/hy/data/ordinal/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hy/data/ordinal/digit.tsv b/nemo_text_processing/text_normalization/hy/data/ordinal/digit.tsv new file mode 100644 index 000000000..92322351a --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/ordinal/digit.tsv @@ -0,0 +1,5 @@ +հինգերորդ հինգ +վեցերորդ վեց +յոթերորդ յոթ +ութերորդ ութ +իններորդ ինը \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/time/__init__.py b/nemo_text_processing/text_normalization/hy/data/time/__init__.py new file mode 100644 index 000000000..9df65818d --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hy/data/time/hours.tsv b/nemo_text_processing/text_normalization/hy/data/time/hours.tsv new file mode 100644 index 000000000..01656d4e8 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/time/hours.tsv @@ -0,0 +1,26 @@ +մեկ 01 +երկուսն 02 +երեք 03 +չորս 04 +հինգ 05 +վեց 06 +յոթ 07 +ութ 08 +ինն 09 +տասն 10 +տասնմեկ 11 +տասնմեկն 11 +տասներկուսն 12 +տասներեք 13 +տասնչորս 14 +տասնհինգ 15 +տասնվեց 16 +տասնյոթ 17 +տասնութ 18 +տասնինն 19 +քսան 20 +քսանմեկ 21 +քսաներկուսն 22 +քսաներեք 23 +քսանչորս 24 +քսանչորս 24 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/time/minutes.tsv b/nemo_text_processing/text_normalization/hy/data/time/minutes.tsv new file mode 100644 index 000000000..5035ad406 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/time/minutes.tsv @@ -0,0 +1,61 @@ +մեկ 01 +երկու 02 +երեք 03 +չորս 04 +հինգ 05 +վեց 06 +յոթ 07 +ութ 08 +ինը 09 +տաս 10 +տասնմեկ 11 +տասնմեկն 11 +տասներկու 12 +տասներեք 13 +տասնչորս 14 +տասնհինգ 15 +տասնվեց 16 +տասնյոթ 17 +տասնութ 18 +տասնինը 19 +քսան 20 +քսանմեկ 21 +քսաներկու 22 +քսաներեք 23 +քսանչորս 24 +քսանհինգ 25 +քսանվեց 26 +քսանյոթ 27 +քսանութ 28 +քսանինը 29 +երեսուն 30 +երեսունմեկ 31 +երեսուներկու 32 +երեսուներեք 33 +երեսունչորս 34 +երեսունհինգ 35 +երեսունվեց 36 +երեսունյոթ 37 +երեսունութ 38 +երեսունինը 39 +քառասուն 41 +քառասունմեկ 41 +քառասուներկու 42 +քառասուներեք 43 +քառասունչորս 44 +քառասունհինգ 45 +քառասունվեց 46 +քառասունյոթ 47 +քառասունութ 48 +քառասունինը 49 +հիսուն 50 +հիսունմեկ 51 +հիսուներկու 52 +հիսուներեք 53 +հիսունչորս 54 +հիսունհինգ 55 +հիսունվեց 56 +հիսունյոթ 57 +հիսունութ 58 +հիսունինը 59 +զրո֊զրո 00 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hy/data/whitelist.tsv b/nemo_text_processing/text_normalization/hy/data/whitelist.tsv new file mode 100644 index 000000000..8ef036e32 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/data/whitelist.tsv @@ -0,0 +1,14 @@ +ս.թ. սույն թվականի +մ.թ.ա. մեր թվարկությունից առաջ +մ.թ. մեր թվարկություն +Ք.ա. քրիստոսից առաջ +Ք.հ. քրիստոսից հետո +Ք.ծ.ա. քրիստոսի ծննդից առաջ +Ք.ծ.հ. քրիստոսի ծննդից հետո +ս․թ․ սույն թվականի +մ․թ․ա․ մեր թվարկությունից առաջ +մ․թ․ մեր թվարկություն +Ք․ա․ քրիստոսից առաջ +Ք․հ․ քրիստոսից հետո +Ք․ծ․ա․ քրիստոսի ծննդից առաջ +Ք․ծ․հ․ քրիստոսի ծննդից հետո diff --git a/nemo_text_processing/text_normalization/hy/taggers/__init__.py b/nemo_text_processing/text_normalization/hy/taggers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hy/taggers/cardinal.py b/nemo_text_processing/text_normalization/hy/taggers/cardinal.py new file mode 100644 index 000000000..8a418f117 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/cardinal.py @@ -0,0 +1,88 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.hy.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals, e.g. + 55 -> cardinal { integer: "հիսունհինգ" } + """ + + def __init__(self): + super().__init__(name="cardinal", kind="classify") + + zero = pynini.string_map([("0", "զրո")]) + digits = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + digits_no_one = (NEMO_DIGIT - "1") @ digits + + ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).invert() + ties_unit = digits + double_digits = (pynini.cross("1", "տասը") | ties) + pynutil.delete("0") | ( + pynini.cross("1", "տասն") | ties + ) + ties_unit + + self.all_double_digits = double_digits.optimize() + + one_to_all_tens = digits | double_digits + self.one_to_all_tens = one_to_all_tens.optimize() + + hundreds_parts = (pynutil.delete("0") + insert_space + digits) | (insert_space + double_digits) + one_hundreds = pynini.cross("1", "հարյուր") + (pynutil.delete("00") | hundreds_parts) + multiple_hundreds = (digits_no_one + insert_space + pynutil.insert("հարյուր")) + ( + pynutil.delete("00") | hundreds_parts + ) + all_hundreds = one_hundreds | multiple_hundreds + self.all_hundreds = all_hundreds.optimize() + + delete_separator = pynini.closure(delete_space, 0, 1) + one_thousand = pynini.cross("1", "հազար") + delete_separator + other_thousands = ( + (digits_no_one | double_digits | all_hundreds) + insert_space + pynutil.insert("հազար") + delete_separator + ) + all_thousands = ( + ((one_thousand | other_thousands) + pynutil.delete("000")) + | (one_thousand + pynutil.delete("00") + insert_space + digits) + | (other_thousands + pynutil.delete("00") + insert_space + digits) + | ((one_thousand | other_thousands) + pynutil.delete("0") + insert_space + double_digits) + | ((one_thousand | other_thousands) + insert_space + all_hundreds) + ) + + digits_to_hundreds = digits | double_digits | all_hundreds + digits_to_thousands = digits | double_digits | all_hundreds | all_thousands + millions_components = pynini.closure(delete_separator + pynini.closure(NEMO_DIGIT, 3), 2) + delete_zeros = pynini.closure(pynutil.delete("0"), 0, 6) + all_millions = (digits_to_hundreds + insert_space + pynutil.insert("միլիոն")) + ( + millions_components @ (delete_zeros + pynini.closure(insert_space + digits_to_thousands, 0, 1)) + ) + + digits_to_millions = digits_to_thousands | all_millions + billions_components = pynini.closure(delete_separator + pynini.closure(NEMO_DIGIT, 3), 3) + delete_zeros = pynini.closure(pynutil.delete("0"), 0, 9) + all_billions = (digits_to_hundreds + insert_space + pynutil.insert("միլիարդ")) + ( + billions_components @ (delete_zeros + pynini.closure(insert_space + digits_to_millions, 0, 1)) + ) + + final_graph = zero | digits | double_digits | all_hundreds | all_thousands | all_millions | all_billions + self.all_nums_no_tokens = final_graph + + final_graph = pynutil.insert("integer: \"") + final_graph + pynutil.insert("\"") + self.final_graph = final_graph + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/decimal.py b/nemo_text_processing/text_normalization/hy/taggers/decimal.py new file mode 100644 index 000000000..ad9474670 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/decimal.py @@ -0,0 +1,87 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.hy.utils import get_abs_path + + +def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": + """ + Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, + e.g. 2 միլիոն -> integer_part: "երկու" quantity: "միլիոն" + e.g. 2․4 միլիոն -> integer_part: "երկու" fractional_part: "չորս" quantity: "միլիոն" + Args: + decimal_graph: DecimalFST + cardinal_graph: CardinalFST + """ + quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) + delete_separator = pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1) + numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph + numbers = pynini.cdrewrite(pynutil.delete(delete_separator), "", "", NEMO_SIGMA) @ numbers + + res = ( + pynutil.insert('integer_part: "') + + numbers + + pynutil.insert('"') + + NEMO_SPACE + + pynutil.insert('quantity: "') + + quantities + + pynutil.insert('"') + ) + res |= decimal_graph + NEMO_SPACE + pynutil.insert('quantity: "') + quantities + pynutil.insert('"') + return res + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + 554 միլիարդ -> decimal { integer_part: "հինգ հարյուր հիսունչորս" quantity: "միլիարդ" } + Args: + cardinal: CardinalFst + deterministic is not necessary right now + TODO make deterministic make sense + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + graph = cardinal.one_to_all_tens + + graph = graph.optimize() + + delete_separator = pynutil.delete(".") | pynutil.delete("․") + optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + graph_fractional = pynutil.insert('fractional_part: "') + graph + pynutil.insert('"') + + integers = cardinal.all_nums_no_tokens + graph_integer = pynutil.insert('integer_part: "') + integers + pynutil.insert('"') + final_graph_wo_sign = graph_integer + delete_separator + insert_space + graph_fractional + + final_graph_wo_negative = final_graph_wo_sign | get_quantity(final_graph_wo_sign, integers) + self.final_graph_wo_negative = final_graph_wo_negative.optimize() + + final_graph = optional_graph_negative + final_graph_wo_negative + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/fraction.py b/nemo_text_processing/text_normalization/hy/taggers/fraction.py new file mode 100644 index 000000000..6cf4c1fee --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/fraction.py @@ -0,0 +1,37 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, GraphFst + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fraction, e.g. + "4/5" -> tokens { fraction { numerator: "չորս" denominator: "հինգերորդ" } } + "1/6" -> tokens { fraction { numerator: "մեկ" denominator: "վեցերորդ" } } + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="fraction", kind="classify") + cardinal_graph = cardinal.all_nums_no_tokens + ordinal_graph = ordinal.denominator_graph + + numerator = pynutil.insert("numerator: \"") + cardinal_graph + pynutil.insert("\"") + denominator = pynutil.insert(" denominator: \"") + ordinal_graph + pynutil.insert("\"") + + final_graph = numerator + pynutil.delete("/") + denominator + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/measure.py b/nemo_text_processing/text_normalization/hy/taggers/measure.py new file mode 100644 index 000000000..004b76c07 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/measure.py @@ -0,0 +1,99 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + NEMO_SIGMA, + TO_LOWER, + GraphFst, + delete_extra_space, +) +from nemo_text_processing.text_normalization.hy.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure + e.g. 52 կգ -> measure { cardinal { integer: "հիսուներկու" } units: "կիլոգրամ" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="measure", kind="classify") + + cardinal_graph = cardinal.all_nums_no_tokens + + casing_graph = pynini.closure(TO_LOWER | NEMO_SIGMA) + + graph_measurements_unit = pynini.string_file(get_abs_path("data/measurements.tsv")) + graph_measurements_unit = pynini.compose(casing_graph, graph_measurements_unit) + + graph_measurements_dates_unit = pynini.string_file(get_abs_path("data/measurement_dates.tsv")) + graph_measurements_dates_unit = pynini.compose(casing_graph, graph_measurements_dates_unit) + + measurements_unit = pynutil.insert("units: \"") + graph_measurements_unit + pynutil.insert("\"") + + measurements_dates_unit = pynutil.insert("units: \"") + graph_measurements_dates_unit + pynutil.insert("\"") + + subgraph_decimal = ( + pynutil.insert("decimal { ") + + decimal.final_graph_wo_negative + + pynutil.insert(" }") + + pynini.closure(delete_extra_space, 0, 1) + + measurements_unit + ) + + subgraph_cardinal = ( + pynutil.insert("cardinal { ") + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + pynini.closure(delete_extra_space, 0, 1) + + measurements_unit + ) + + subgraph_cardinal_dates = ( + pynutil.insert("cardinal { ") + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + pynini.closure(delete_extra_space, 0, 1) + + measurements_dates_unit + ) + + subgraph_cardinal_dates |= ( + pynutil.insert("cardinal { ") + + pynutil.insert("integer: \"") + + cardinal_graph + + pynutil.insert("ից") + + pynutil.delete("-") + + pynutil.insert(' ') + + cardinal_graph + + pynutil.insert("\"") + + pynutil.insert(" }") + + pynini.closure(delete_extra_space, 0, 1) + + measurements_dates_unit + ) + + final_graph = subgraph_decimal | subgraph_cardinal | subgraph_cardinal_dates + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/money.py b/nemo_text_processing/text_normalization/hy/taggers/money.py new file mode 100644 index 000000000..e84bb5b0c --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/money.py @@ -0,0 +1,83 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space +from nemo_text_processing.text_normalization.hy.utils import get_abs_path + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + "15 $" -> money { "տասնհինգ դոլար" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.final_graph + decimal_graph = decimal.fst + + unit = pynini.string_file(get_abs_path("data/currency.tsv")) + + weighted_delimiter = pynutil.add_weight(pynutil.delete(NEMO_SPACE), -100) + optional_delimiter = pynini.closure(weighted_delimiter, 0, 1) + graph_unit_singular = optional_delimiter + pynutil.insert(" currency: \"") + unit + pynutil.insert("\"") + + graph_decimal = decimal_graph + graph_unit_singular + graph_cardinal = cardinal_graph + graph_unit_singular + + tagger_graph = graph_cardinal | graph_decimal + + integer = pynutil.delete("\"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + integer_cardinal = pynutil.delete("integer: ") + integer + integer_part = pynutil.delete("integer_part: ") + integer + + unit = ( + pynutil.delete("currency: ") + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + unit = pynini.accep(NEMO_SPACE) + unit + + verbalizer_graph_cardinal = integer_cardinal + unit + + optional_fractional_part = pynini.closure(pynutil.delete("fractional_part: ") + integer, 0, 1) + optional_quantity = pynini.closure(pynini.accep(NEMO_SPACE) + pynutil.delete("quantity: ") + integer, 0, 1) + + verbalizer_graph_decimal = ( + pynutil.delete('decimal { ') + + integer_part + + delete_space + + pynutil.insert(" ամբողջ ") + + optional_fractional_part + + delete_space + + optional_quantity + + delete_space + + pynutil.delete(" }") + + unit + ) + + verbalizer_graph = verbalizer_graph_cardinal | verbalizer_graph_decimal + + self.final_graph = (tagger_graph @ verbalizer_graph).optimize() + self.fst = self.add_tokens( + pynutil.insert("integer_part: \"") + self.final_graph + pynutil.insert("\"") + ).optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/ordinal.py b/nemo_text_processing/text_normalization/hy/taggers/ordinal.py new file mode 100644 index 000000000..ec486bd92 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/ordinal.py @@ -0,0 +1,74 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal, e.g. + 5-րդ -> ordinal { integer: "հինգերորդ" } + 1-ին -> ordinal { integer: "առաջին" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.all_nums_no_tokens + cardinal_format = pynini.closure(NEMO_DIGIT | pynini.accep(",")) + rd = pynini.accep("-րդ") + first_format = ( + pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + pynini.accep("1") + pynutil.delete("-ին") + ) + second_format = pynini.closure(cardinal_format + (NEMO_DIGIT - "2"), 0, 1) + pynini.accep("2") + third_format = pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + pynini.accep("3") + fourth_format = pynini.closure(cardinal_format + (NEMO_DIGIT - "1"), 0, 1) + pynini.accep("4") + th_format = pynini.closure( + (NEMO_DIGIT - "1" - "2" - "3" - "4") | (cardinal_format + "1" + NEMO_DIGIT) | cardinal_format, 1 + ) + + first = pynini.cross("1", "առաջին") + second = pynini.cross("2", "երկրորդ") + third = pynini.cross("3", "երրորդ") + fourth = pynini.cross("4", "չորրորդ") + + special_denominator_graph = second_format @ second | third_format @ third | fourth_format @ fourth + + self.denominator_graph = ( + pynutil.add_weight(first_format @ first, 1) + | pynutil.add_weight(special_denominator_graph, 1) + | pynutil.add_weight(th_format @ cardinal_graph + pynutil.insert("երորդ"), 1.5) + ).optimize() + + special_ordinals_graph = ( + (second_format + pynutil.delete(rd)) @ second + | (third_format + pynutil.delete(rd)) @ third + | (fourth_format + pynutil.delete(rd)) @ fourth + ) + + self.graph = ( + pynutil.add_weight(first_format @ first, 1) + | pynutil.add_weight(special_ordinals_graph, 1) + | pynutil.add_weight((th_format + pynutil.delete(rd)) @ cardinal_graph + pynutil.insert("երորդ"), 1.5) + ).optimize() + + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/punctuation.py b/nemo_text_processing/text_normalization/hy/taggers/punctuation.py new file mode 100644 index 000000000..8f6852cbd --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/punctuation.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. , -> tokens { name: "," } + + """ + + def __init__(self): + super().__init__(name="punctuation", kind="classify") + + s = "!#$%&\'()*+,-./:;<=>?@^_`{|}~" + punct = pynini.union(*s) + + graph = pynutil.insert("name: \"") + punct + pynutil.insert("\"") + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/time.py b/nemo_text_processing/text_normalization/hy/taggers/time.py new file mode 100644 index 000000000..5c555456e --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/time.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED, GraphFst +from nemo_text_processing.text_normalization.hy.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + "15:52" -> time { hours: "տասնհինգ" minutes: "հիսուներկու" } + """ + + def __init__(self, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="time", kind="classify") + hours = pynini.string_file(get_abs_path('data/time/hours.tsv')).invert() + minutes = pynini.string_file(get_abs_path('data/time/minutes.tsv')).invert() + + graph_hours = pynutil.insert("hours: \"") + hours + pynutil.insert("\"") + graph_minutes = pynutil.insert(" minutes: \"") + minutes + pynutil.insert("\"") + + final_graph = graph_hours + pynutil.delete(":") + graph_minutes + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/hy/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..a4562b436 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/tokenize_and_classify.py @@ -0,0 +1,116 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + INPUT_LOWER_CASED, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.hy.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.hy.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.hy.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.hy.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.hy.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.hy.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.hy.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.hy.taggers.time import TimeFst +from nemo_text_processing.text_normalization.hy.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.hy.taggers.word import WordFst +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__( + self, + cache_dir: str = None, + whitelist: str = None, + deterministic: bool = False, + overwrite_cache: bool = False, + input_case: str = INPUT_LOWER_CASED, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"_hy_tn_{input_case}.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + + ordinal = OrdinalFst(cardinal) + ordinal_graph = ordinal.fst + + fraction = FractionFst(cardinal=cardinal, ordinal=ordinal) + fraction_graph = fraction.fst + + decimal = DecimalFst(cardinal) + decimal_graph = decimal.fst + + measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst + word_graph = WordFst().fst + time_graph = TimeFst().fst + money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst + punct_graph = PunctuationFst().fst + whitelist_graph = WhiteListFst( + input_case=input_case, deterministic=deterministic, input_file=whitelist + ).fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(measure_graph, 0.9) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.09) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/nemo_text_processing/text_normalization/hy/taggers/whitelist.py b/nemo_text_processing/text_normalization/hy/taggers/whitelist.py new file mode 100644 index 000000000..4b68bc011 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/whitelist.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space, load_labels +from nemo_text_processing.text_normalization.hy.utils import get_abs_path + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist, e.g. + մ.թ.ա. -> tokens { name: "մեր թվարկությունից առաջ" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify") + + def _get_whitelist_graph(file): + whitelist = load_labels(file) + whitelist = [[x, y] for x, y in whitelist] + + graph = pynini.string_map(whitelist) + return graph + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + if input_file: + whitelist_provided = _get_whitelist_graph(input_file) + whitelist |= whitelist_provided + + graph = pynutil.insert("name: \"") + convert_space(whitelist) + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/taggers/word.py b/nemo_text_processing/text_normalization/hy/taggers/word.py new file mode 100644 index 000000000..fcacc6c48 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/taggers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. Considers sentence boundary exceptions. + e.g. բարև -> tokens { name: "բարև" } + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/hy/utils.py b/nemo_text_processing/text_normalization/hy/utils.py new file mode 100644 index 000000000..26c9f5119 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/utils.py @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path) + labels = list(csv.reader(label_tsv, delimiter="\t")) + label_tsv.close() + return labels diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/__init__.py b/nemo_text_processing/text_normalization/hy/verbalizers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/hy/verbalizers/cardinal.py new file mode 100644 index 000000000..9b4eeb588 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/cardinal.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { integer: "հիսունհինգ" } -> հիսունհինգ + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + number = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + final_graph = number + self.numbers = final_graph + self.fst = self.delete_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/decimal.py b/nemo_text_processing/text_normalization/hy/verbalizers/decimal.py new file mode 100644 index 000000000..bdea252e8 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/decimal.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { integer_part: "հինգ" quantity: "միլիոն" } -> 5 միլիոն + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + fractional_default = ( + pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + + conjunction = pynutil.insert(" ամբողջ ") + fractional = conjunction + fractional_default + + quantity = ( + delete_space + + insert_space + + pynutil.delete("quantity: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + optional_quantity = pynini.closure(quantity, 0, 1) + + graph = pynini.union((integer + quantity), (integer + delete_space + fractional + optional_quantity)) + + self.numbers_only_quantity = pynini.union( + (integer + quantity), (integer + delete_space + fractional + quantity) + ).optimize() + + graph += delete_preserve_order + self.numbers = graph + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/fraction.py b/nemo_text_processing/text_normalization/hy/verbalizers/fraction.py new file mode 100644 index 000000000..ff7dbd259 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/fraction.py @@ -0,0 +1,44 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fraction + e.g. fraction { numerator: "երկու" denominator: "երրորդ" } } -> 2/3 + + """ + + def __init__(self): + super().__init__(name="fraction", kind="verbalize") + numerator = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + denominator = ( + pynutil.insert(' ') + + pynutil.delete("denominator: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + suffix = pynini.cdrewrite(pynini.cross("ըերորդ", "ներորդ"), "", "[EOS]", NEMO_SIGMA).optimize() + + graph = (numerator + delete_space + pynini.compose(denominator, suffix)).optimize() + self.numbers = graph.optimize() + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/measure.py b/nemo_text_processing/text_normalization/hy/verbalizers/measure.py new file mode 100644 index 000000000..a844f643e --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/measure.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_NOT_QUOTE, GraphFst, delete_space + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure, e.g. + measure { cardinal { integer: "վաթսուն" } units: "կիլոգրամ" } -> վաթսուն կիլոգրամ + + Args: + decimal: DecimalFst + cardinal: CardinalFst + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst): + super().__init__(name="measure", kind="verbalize") + optional_sign = pynini.closure(pynini.cross("negative: \"true\"", "-"), 0, 1) + unit = ( + pynutil.delete("units:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + ) + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + optional_sign + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_cardinal_first = ( + pynutil.delete("cardinal {") + + delete_space + + optional_sign + + delete_space + + cardinal.numbers + + delete_space + + pynutil.delete("}") + ) + graph_cardinal_two = ( + pynutil.delete("cardinal {") + + pynutil.delete(" integer: \"") + + delete_space + + optional_sign + + delete_space + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("} ") + ) + graph_first = (graph_cardinal_first | graph_decimal) + delete_space + pynutil.insert(" ") + unit + graph_second = graph_cardinal_two + delete_space + pynutil.insert(" ") + unit + graph = graph_first | graph_second + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/money.py b/nemo_text_processing/text_normalization/hy/verbalizers/money.py new file mode 100644 index 000000000..f734278ca --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/money.py @@ -0,0 +1,39 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { "երկու դոլար"" } -> երկու դոլար" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + delete_tokens = self.delete_tokens( + pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + ) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/hy/verbalizers/ordinal.py new file mode 100644 index 000000000..73ed4c918 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/ordinal.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal, e.g. + ordinal { integer: "տասնչորս" } } -> տասնչորսերորդ + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + suffix = pynini.cdrewrite(pynini.cross("ըերորդ", "ներորդ"), "", "[EOS]", NEMO_SIGMA).optimize() + self.graph = (pynini.compose(graph, suffix)).optimize() + delete_tokens = self.delete_tokens(self.graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/time.py b/nemo_text_processing/text_normalization/hy/verbalizers/time.py new file mode 100644 index 000000000..94f9f43dc --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/time.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time + e.g. time { hours: "ինն" minutes: "քսաներկու" } -> ինն անց քսաներկու + + """ + + def __init__(self): + super().__init__(name="time", kind="verbalize") + numerator = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + + denominator = ( + pynutil.insert(' անց ') + + pynutil.delete("minutes: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph = (numerator + delete_space + denominator).optimize() + self.numbers = graph.optimize() + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/hy/verbalizers/verbalize.py new file mode 100644 index 000000000..d5d56cf66 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/verbalize.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst +from nemo_text_processing.text_normalization.hy.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.hy.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.hy.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.hy.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.hy.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.hy.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.hy.verbalizers.time import TimeFst +from nemo_text_processing.text_normalization.hy.verbalizers.whitelist import WhiteListFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic=True): + super().__init__(name="verbalize", kind="verbalize") + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + ordinal_graph = OrdinalFst().fst + decimal = DecimalFst() + decimal_graph = decimal.fst + fraction = FractionFst() + fraction_graph = fraction.fst + measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst + money_graph = MoneyFst().fst + time_graph = TimeFst().fst + whitelist_graph = WhiteListFst().fst + graph = ( + time_graph + | fraction_graph + | measure_graph + | money_graph + | ordinal_graph + | decimal_graph + | cardinal_graph + | whitelist_graph + ) + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/hy/verbalizers/verbalize_final.py new file mode 100644 index 000000000..e5afd807e --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/verbalize_final.py @@ -0,0 +1,51 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.text_normalization.hy.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.hy.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "Երևանում" } tokens { name: "ժամը" } tokens { time { hours: "տասներկուսն" minutes: "հիսունհինգ" } } tokens { name: "է" } tokens { name: ":" } -> Երևանում ժամը տասներկուսն անց հիսունհինգ է: + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic=True, cache_dir=None, overwrite_cache=False): + super().__init__(name="verbalize_final", kind="verbalize") + verbalize = VerbalizeFst().fst + word = WordFst().fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + self.fst = graph diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/hy/verbalizers/whitelist.py new file mode 100644 index 000000000..5fd8f7bdb --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/whitelist.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + def __init__(self): + super().__init__(name="whitelist", kind="verbalize") + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite( + pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA + ) # Removes possible null token + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/hy/verbalizers/word.py b/nemo_text_processing/text_normalization/hy/verbalizers/word.py new file mode 100644 index 000000000..9ef23b1d5 --- /dev/null +++ b/nemo_text_processing/text_normalization/hy/verbalizers/word.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing word + e.g. tokens { name: "արթնանալ" } -> արթնանալ + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/it/__init__.py b/nemo_text_processing/text_normalization/it/__init__.py index 6ebc808fa..4bb2b03f6 100644 --- a/nemo_text_processing/text_normalization/it/__init__.py +++ b/nemo_text_processing/text_normalization/it/__init__.py @@ -11,3 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.it.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/it/data/electronic/__init__ .py b/nemo_text_processing/text_normalization/it/data/__init__.py similarity index 100% rename from nemo_text_processing/text_normalization/it/data/electronic/__init__ .py rename to nemo_text_processing/text_normalization/it/data/__init__.py diff --git a/nemo_text_processing/text_normalization/it/data/numbers/__init__ .py b/nemo_text_processing/text_normalization/it/data/electronic/__init__.py similarity index 100% rename from nemo_text_processing/text_normalization/it/data/numbers/__init__ .py rename to nemo_text_processing/text_normalization/it/data/electronic/__init__.py diff --git a/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv b/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv index 94f60f68d..c5c26d736 100644 --- a/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv +++ b/nemo_text_processing/text_normalization/it/data/electronic/domain.tsv @@ -10,4 +10,5 @@ .us punto US .de punto DE .it punto IT -.jpg punto jpeg \ No newline at end of file +.jpg punto jpeg +.edu punto edu \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv b/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv index 68b808dcd..bb359e6a0 100644 --- a/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv +++ b/nemo_text_processing/text_normalization/it/data/measure/measurements.tsv @@ -60,4 +60,5 @@ l litro dl decilitro bar bar kcal chilocaloria -cal caloria \ No newline at end of file +cal caloria +% percento \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/it/taggers/__init__ .py b/nemo_text_processing/text_normalization/it/data/numbers/__init__.py similarity index 100% rename from nemo_text_processing/text_normalization/it/taggers/__init__ .py rename to nemo_text_processing/text_normalization/it/data/numbers/__init__.py diff --git a/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv b/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv new file mode 100644 index 000000000..1cd570163 --- /dev/null +++ b/nemo_text_processing/text_normalization/it/data/numbers/tens_eight.tsv @@ -0,0 +1,8 @@ +ventotto 28 +trentotto 38 +quarantotto 48 +cinquantotto 58 +sessantotto 68 +settantotto 78 +ottantotto 88 +novantotto 98 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/it/verbalizers/__init__ .py b/nemo_text_processing/text_normalization/it/data/whitelist/__init__.py similarity index 100% rename from nemo_text_processing/text_normalization/it/verbalizers/__init__ .py rename to nemo_text_processing/text_normalization/it/data/whitelist/__init__.py diff --git a/nemo_text_processing/text_normalization/it/data/whitelist.tsv b/nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv similarity index 100% rename from nemo_text_processing/text_normalization/it/data/whitelist.tsv rename to nemo_text_processing/text_normalization/it/data/whitelist/whitelist.tsv diff --git a/nemo_text_processing/text_normalization/it/taggers/__init__.py b/nemo_text_processing/text_normalization/it/taggers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/it/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/it/taggers/cardinal.py b/nemo_text_processing/text_normalization/it/taggers/cardinal.py index 3894004df..1e16d6e36 100644 --- a/nemo_text_processing/text_normalization/it/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/it/taggers/cardinal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -25,13 +27,13 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import cardinal_separator from nemo_text_processing.text_normalization.it.utils import get_abs_path -from pynini.lib import pynutil zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))) tens = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens.tsv"))) tens_one = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens_one.tsv"))) +tens_eight = pynini.invert(pynini.string_file(get_abs_path("data/numbers/tens_eight.tsv"))) hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv"))) @@ -46,7 +48,7 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': Returns: fst: A pynini.FstLike object """ - exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string cardinal_string = pynini.closure( @@ -83,7 +85,8 @@ def __init__(self, deterministic: bool = True): # double digit graph_tens = teen graph_tens |= tens + (pynutil.delete('0') | graph_digit) - graph_tens |= tens_one + graph_tens |= pynutil.add_weight(tens_one, -0.01) + graph_tens |= pynutil.add_weight(tens_eight, -0.01) self.tens = graph_tens.optimize() @@ -159,7 +162,7 @@ def __init__(self, deterministic: bool = True): self.graph = ( ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/it/taggers/decimals.py b/nemo_text_processing/text_normalization/it/taggers/decimals.py index fe1c18e49..8f98d5a2b 100644 --- a/nemo_text_processing/text_normalization/it/taggers/decimals.py +++ b/nemo_text_processing/text_normalization/it/taggers/decimals.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -27,7 +29,6 @@ strip_cardinal_apocope, ) from nemo_text_processing.text_normalization.it.utils import get_abs_path -from pynini.lib import pynutil quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv")) digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) @@ -35,8 +36,7 @@ def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike") -> "pynini.FstLike": - """ - """ + """ """ numbers = pynini.closure(NEMO_DIGIT, 1, 6) @ cardinal_graph numbers = pynini.cdrewrite(pynutil.delete(cardinal_separator), "", "", NEMO_SIGMA) @ numbers diff --git a/nemo_text_processing/text_normalization/it/taggers/electronic.py b/nemo_text_processing/text_normalization/it/taggers/electronic.py index 7bb22c9d0..c28a205ec 100644 --- a/nemo_text_processing/text_normalization/it/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/it/taggers/electronic.py @@ -13,12 +13,28 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space -from nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels from pynini.lib import pynutil -common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] -symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + at, + colon, + domain_string, + double_quotes, + double_slash, + http, + https, + period, + protocol_string, + username_string, + www, +) +from nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels + +# common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] class ElectronicFst(GraphFst): @@ -36,44 +52,55 @@ def __init__(self, deterministic: bool = True): super().__init__(name="electronic", kind="classify", deterministic=deterministic) dot = pynini.accep(".") - accepted_common_domains = pynini.union(*common_domains) - accepted_symbols = pynini.union(*symbols) - dot - accepted_characters = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols) - acceepted_characters_with_dot = pynini.closure(NEMO_ALPHA | NEMO_DIGIT | accepted_symbols | dot) - # e-mail - username = ( - pynutil.insert("username: \"") - + acceepted_characters_with_dot - + pynutil.insert("\"") - + pynini.cross('@', ' ') - ) + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_period = pynini.difference(symbols, dot) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_period), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) - domain_graph = accepted_characters + dot + accepted_characters + # domains + domain = dot + accepted_characters domain_graph = ( - pynutil.insert("domain: \"") - + domain_graph - + pynini.closure((accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1) - + pynutil.insert("\"") + pynutil.insert(domain_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters + pynini.closure(domain, 1)) + + pynutil.insert(double_quotes) ) - domain_common_graph = ( - pynutil.insert("domain: \"") - + accepted_characters - + accepted_common_domains - + pynini.closure((accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1) - + pynutil.insert("\"") + # email + username = ( + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + all_characters + + pynutil.insert(double_quotes) + + pynini.cross(at, NEMO_SPACE) ) + email = username + domain_graph - graph = (username + domain_graph) | domain_common_graph + # social media tags + tag = ( + pynutil.delete(at) + + pynutil.insert(username_string + colon + NEMO_SPACE + double_quotes) + + (accepted_characters | (accepted_characters + pynini.closure(domain, 1))) + + pynutil.insert(double_quotes) + ) - protocol_start = pynini.accep("https://") | pynini.accep("http://") + # url + protocol_start = pynini.accep(https + colon + double_slash) | pynini.accep(http + colon + double_slash) protocol_end = ( - pynini.accep("www.") if deterministic else pynini.accep("www.") | pynini.cross("www.", "vu vu vu.") + pynini.accep(www + period) + if deterministic + else pynini.accep(www + period) | pynini.cross((www + period), "vu vu vu.") ) protocol = protocol_start | protocol_end | (protocol_start + protocol_end) - protocol = pynutil.insert("protocol: \"") + protocol + pynutil.insert("\"") - graph |= protocol + insert_space + (domain_graph | domain_common_graph) + protocol = ( + pynutil.insert(protocol_string + colon + NEMO_SPACE + double_quotes) + + protocol + + pynutil.insert(double_quotes) + ) + + url = protocol + pynutil.insert(NEMO_SPACE) + (domain_graph) + + graph = url | domain_graph | email | tag self.graph = graph final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) diff --git a/nemo_text_processing/text_normalization/it/taggers/measure.py b/nemo_text_processing/text_normalization/it/taggers/measure.py index c287ada52..880be0aa7 100644 --- a/nemo_text_processing/text_normalization/it/taggers/measure.py +++ b/nemo_text_processing/text_normalization/it/taggers/measure.py @@ -13,6 +13,9 @@ # limitations under the License. import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -23,8 +26,6 @@ insert_space, ) from nemo_text_processing.text_normalization.it.utils import get_abs_path -from pynini.examples import plurals -from pynini.lib import pynutil unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv")) suppletive = pynini.string_file(get_abs_path("data/measure/suppletive.tsv")) @@ -44,7 +45,7 @@ class MeasureFst(GraphFst): """ Finite state transducer for classifying measure, e.g. "2,4 g" -> measure { cardinal { integer_part: "due" fractional_part: "quattro" units: "grammi" preserve_order: true } } - + Args: cardinal: CardinalFst decimal: DecimalFst @@ -67,7 +68,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = ) optional_unit_denominator = pynini.closure( - pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, 0, 1, + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit_denominator, + 0, + 1, ) unit_plural = ( diff --git a/nemo_text_processing/text_normalization/it/taggers/money.py b/nemo_text_processing/text_normalization/it/taggers/money.py index 70647096e..d92906f1d 100644 --- a/nemo_text_processing/text_normalization/it/taggers/money.py +++ b/nemo_text_processing/text_normalization/it/taggers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import decimal_separator from nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels -from pynini.lib import pynutil maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) @@ -39,7 +40,7 @@ class MoneyFst(GraphFst): "€1" -> money { currency_maj: "euro" integer_part: "un"} "€1,000" -> money { currency_maj: "euro" integer_part: "un" } "4,2 £" -> money { integer_part: "quattro" currency_maj: "sterline" fractional_part: "venti" currency_min: "penny" preserve_order: true } - + Args: cardinal: CardinalFst decimal: DecimalFst diff --git a/nemo_text_processing/text_normalization/it/taggers/time.py b/nemo_text_processing/text_normalization/it/taggers/time.py new file mode 100644 index 000000000..97d952489 --- /dev/null +++ b/nemo_text_processing/text_normalization/it/taggers/time.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 15:30:30 tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e mezza trenta secondi + 12:15 tokens { time { hours: "12" minutes: "15" } } -> dodici e un quarto + 03:38 tokens { time { hours: "3" minutes: "38" } } -> tre e trentotto minuti + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + labels_hour = [str(x) for x in range(0, 25)] + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + delete_leading_zero_to_double_digit = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT + + graph_hour = pynini.union(*labels_hour) + + graph_minute_single = pynini.union(*labels_minute_single) + graph_minute_double = pynini.union(*labels_minute_double) + + final_graph_hour_only = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"") + final_graph_hour = ( + pynutil.insert("hours: \"") + delete_leading_zero_to_double_digit @ graph_hour + pynutil.insert("\"") + ) + final_graph_minute = ( + pynutil.insert("minutes: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + final_graph_second = ( + pynutil.insert("seconds: \"") + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert("\"") + ) + + graph_hm = ( + final_graph_hour + pynutil.delete(":") + (pynutil.delete("00") | (insert_space + final_graph_minute)) + ) + + graph_hms = ( + final_graph_hour + + pynutil.delete(":") + + (pynini.cross("00", " minutes: \"0\"") | (insert_space + final_graph_minute)) + + pynutil.delete(":") + + (pynini.cross("00", " seconds: \"0\"") | (insert_space + final_graph_second)) + + pynutil.insert(" preserve_order: true") + ) + + graph_h = final_graph_hour_only + + final_graph = (graph_hm | graph_h | graph_hms).optimize() + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py index f57c248e9..603d520b5 100644 --- a/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/it/taggers/tokenize_and_classify.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_CHAR, + NEMO_DIGIT, NEMO_WHITE_SPACE, GraphFst, delete_extra_space, @@ -29,9 +32,10 @@ from nemo_text_processing.text_normalization.it.taggers.electronic import ElectronicFst from nemo_text_processing.text_normalization.it.taggers.measure import MeasureFst from nemo_text_processing.text_normalization.it.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.it.taggers.time import TimeFst from nemo_text_processing.text_normalization.it.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.it.taggers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -62,13 +66,14 @@ def __init__( os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" far_file = os.path.join( - cache_dir, f"_{input_case}_it_tn_{deterministic}_deterministic{whitelist_file}.far" + cache_dir, + f"_{input_case}_it_tn_{deterministic}_deterministic{whitelist_file}.far", ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") self.cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = self.cardinal.fst @@ -84,48 +89,50 @@ def __init__( self.electronic = ElectronicFst(deterministic=deterministic) electronic_graph = self.electronic.fst - self.measure = MeasureFst(cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic) + self.measure = MeasureFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) measure_graph = self.measure.fst - self.money = MoneyFst(cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic) + self.money = MoneyFst( + cardinal=self.cardinal, + decimal=self.decimal, + deterministic=deterministic, + ) money_graph = self.money.fst + self.time = TimeFst(deterministic=deterministic) + time_graph = self.time.fst + punct_graph = PunctuationFst(deterministic=deterministic).fst classify = ( - pynutil.add_weight(whitelist_graph, 1) + pynutil.add_weight(whitelist_graph, 0.0) | pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(decimal_graph, 1.1) - | pynutil.add_weight(electronic_graph, 1.09) - | pynutil.add_weight(measure_graph, 1.09) - | pynutil.add_weight(money_graph, 1.09) + | pynutil.add_weight(electronic_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) | pynutil.add_weight(word_graph, 100) ) - punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") - punct = pynini.closure( - pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) - | (pynutil.insert(" ") + punct), - 1, - ) + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") token_plus_punct = ( pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) ) - graph = token_plus_punct + pynini.closure( - ( - pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) - | (pynutil.insert(" ") + punct + pynutil.insert(" ")) - ) - + token_plus_punct - ) + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) graph = delete_space + graph + delete_space - graph |= punct self.fst = graph.optimize() + no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) + self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/it/taggers/whitelist.py b/nemo_text_processing/text_normalization/it/taggers/whitelist.py index 776d03512..c2f5c17db 100644 --- a/nemo_text_processing/text_normalization/it/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/it/taggers/whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.it.utils import get_abs_path, load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): @@ -42,10 +43,10 @@ def _get_whitelist_graph(input_case, file): graph = pynini.string_map(whitelist) return graph - graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/whitelist.tsv")) if not deterministic and input_case != "lower_cased": graph |= pynutil.add_weight( - _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist/whitelist.tsv")), weight=0.0001 ) if input_file: @@ -56,7 +57,7 @@ def _get_whitelist_graph(input_case, file): graph = whitelist_provided if not deterministic: - units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measures/measurements.tsv")) + units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measure/measurements.tsv")) graph |= units_graph self.graph = graph diff --git a/nemo_text_processing/text_normalization/it/taggers/word.py b/nemo_text_processing/text_normalization/it/taggers/word.py index de3f4b7d3..b6746f79d 100644 --- a/nemo_text_processing/text_normalization/it/taggers/word.py +++ b/nemo_text_processing/text_normalization/it/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/it/utils.py b/nemo_text_processing/text_normalization/it/utils.py index 752ff1542..be8bdb5ad 100644 --- a/nemo_text_processing/text_normalization/it/utils.py +++ b/nemo_text_processing/text_normalization/it/utils.py @@ -22,7 +22,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path @@ -37,6 +37,6 @@ def load_labels(abs_path): Returns dictionary of mappings """ - label_tsv = open(abs_path, encoding="utf-8") - labels = list(csv.reader(label_tsv, delimiter="\t")) + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) return labels diff --git a/nemo_text_processing/text_normalization/it/verbalizers/__init__.py b/nemo_text_processing/text_normalization/it/verbalizers/__init__.py new file mode 100644 index 000000000..6ebc808fa --- /dev/null +++ b/nemo_text_processing/text_normalization/it/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py index 30f7b4bcb..0e2b3d8b6 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/it/verbalizers/decimal.py b/nemo_text_processing/text_normalization/it/verbalizers/decimal.py index aa527c9f7..f257d7df4 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/decimal.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -20,23 +22,22 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. - decimal { negative: "true" integer_part: "venti" fractional_part: "trentaquattro" quantity: "miliardi" } -> + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "venti" fractional_part: "trentaquattro" quantity: "miliardi" } -> meno venti virgola trentaquattro - decimal { integer_part: "un milione" fractional_part: "zero zero zero" quantity: "milioni" preserve_order: true } --> + decimal { integer_part: "un milione" fractional_part: "zero zero zero" quantity: "milioni" preserve_order: true } --> un milione virgola zero zero zero decimal { integer_part: "due" quantity: "milioni" preserve_order: true } --> due milioni Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="decimal", kind="classify", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/it/verbalizers/electronic.py b/nemo_text_processing/text_normalization/it/verbalizers/electronic.py index 4925ee5ba..f7071d3d2 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/electronic.py @@ -13,15 +13,21 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import ( +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import ( # Common string literals; expand as you see fit. NEMO_NOT_QUOTE, NEMO_SIGMA, + NEMO_SPACE, GraphFst, + colon, delete_preserve_order, - insert_space, + domain_string, + double_quotes, + protocol_string, + username_string, ) from nemo_text_processing.text_normalization.it.utils import get_abs_path -from pynini.lib import pynutil digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) @@ -34,7 +40,7 @@ class ElectronicFst(GraphFst): """ Finite state transducer for verbalizing electronic - e.g. electronic { username: "abc.def2" domain: "studenti.università.it" } -> + e.g. electronic { username: "abc.def2" domain: "studenti.università.it" } -> "a b c punto d e f due chiocciola s t u d e n t i punto u n i v e r s i t à punto IT Args: deterministic: if True will provide a single transduction option, @@ -47,27 +53,36 @@ def __init__(self, deterministic: bool = True): graph_digit = digit_no_zero | zero def add_space_after_char(): - return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + ( - NEMO_NOT_QUOTE - pynini.accep(" ") + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) ) verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) - user_name = pynutil.delete("username: \"") + add_space_after_char() + pynutil.delete("\"") + user_name = ( + pynutil.delete(username_string + colon + NEMO_SPACE + double_quotes) + + add_space_after_char() + + pynutil.delete(double_quotes) + ) user_name @= verbalize_characters convert_defaults = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | server_common | domain_common - domain = convert_defaults + pynini.closure(insert_space + convert_defaults) + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) domain @= verbalize_characters - domain = pynutil.delete("domain: \"") + domain + pynutil.delete("\"") + domain = ( + pynutil.delete(domain_string + colon + NEMO_SPACE + double_quotes) + domain + pynutil.delete(double_quotes) + ) protocol = ( - pynutil.delete("protocol: \"") + pynutil.delete(protocol_string + colon + NEMO_SPACE + double_quotes) + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) - + pynutil.delete("\"") + + pynutil.delete(double_quotes) ) - self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | ( - user_name + pynini.accep(" ") + pynutil.insert("chiocciola ") + domain + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert("chiocciola ") + domain + | (pynutil.insert("chiocciola ") + user_name) ) + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/it/verbalizers/measure.py b/nemo_text_processing/text_normalization/it/verbalizers/measure.py index 6e93f074c..c7fe33969 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/measure.py @@ -13,20 +13,21 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_preserve_order, ) -from pynini.lib import pynutil class MeasureFst(GraphFst): """ Finite state transducer for verbalizing measure, e.g. measure { cardinal { integer: "due" units: "grammi" } } -> "due grammi" - + Args: decimal: decimal GraphFst cardinal: cardinal GraphFst diff --git a/nemo_text_processing/text_normalization/it/verbalizers/money.py b/nemo_text_processing/text_normalization/it/verbalizers/money.py index 960a1d927..f4b3fdef8 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/money.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -22,7 +24,6 @@ ) from nemo_text_processing.text_normalization.es.graph_utils import shift_cardinal_gender, strip_cardinal_apocope from nemo_text_processing.text_normalization.it.utils import get_abs_path -from pynini.lib import pynutil fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv"))) masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv"))) @@ -39,7 +40,7 @@ class MoneyFst(GraphFst): Finite state transducer for verbalizing money, e.g. money { currency_maj: "euro" integer_part: "un"} -> "un euro" money { integer_part: "quattro" currency_maj: "sterline" fractional_part: "venti" currency_min: "penny" preserve_order: true } -> "quattro sterline venti penny" - + Args: decimal: GraphFst deterministic: if True will provide a single transduction option, diff --git a/nemo_text_processing/text_normalization/it/verbalizers/time.py b/nemo_text_processing/text_normalization/it/verbalizers/time.py new file mode 100644 index 000000000..6f098f152 --- /dev/null +++ b/nemo_text_processing/text_normalization/it/verbalizers/time.py @@ -0,0 +1,85 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_preserve_order +from nemo_text_processing.text_normalization.it.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time, e.g. + tokens { time { hours: "15" minutes: "30" seconds: "30" preserve_order: true } } -> quindici e mezza trenta secondi + tokens { time { hours: "12" minutes: "15" } } -> dodici e un quarto + tokens { time { hours: "3" minutes: "38" } } -> tre e trentotto minuti + Args: + cardinal_tagger: cardinal_tagger tagger GraphFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + graph_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))).optimize() + number_verbalization = graph_zero | cardinal_tagger.two_digit_no_zero + hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + hour_verbalized = hour @ number_verbalization + minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"") + + graph_hms_15 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) + + pynini.accep(" ") + + pynutil.insert("e ") + + second @ number_verbalization + + pynutil.insert(" secondi") + ) + + graph_hms_30 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) + + pynini.accep(" ") + + pynutil.insert("e ") + + second @ number_verbalization + + pynutil.insert(" secondi") + ) + + graph_hm_15 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("15", "un quarto")) + ) + + graph_hm_30 = ( + hour_verbalized + + pynini.accep(" ") + + pynutil.insert("e ") + + (minute @ number_verbalization + pynutil.insert(" minuti") | minute @ pynini.cross("30", " mezza")) + ) + + graph_h = hour_verbalized + + self.graph = graph_hms_30 | graph_hms_30 | graph_hms_15 | graph_hm_30 | graph_hm_15 | graph_h + + delete_tokens = self.delete_tokens(self.graph + delete_preserve_order) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py index 9d8e8acb6..f17f73043 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/verbalize.py @@ -14,11 +14,13 @@ from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.it.taggers.cardinal import CardinalFst as CardinalTagger from nemo_text_processing.text_normalization.it.verbalizers.cardinal import CardinalFst from nemo_text_processing.text_normalization.it.verbalizers.decimal import DecimalFst from nemo_text_processing.text_normalization.it.verbalizers.electronic import ElectronicFst from nemo_text_processing.text_normalization.it.verbalizers.measure import MeasureFst from nemo_text_processing.text_normalization.it.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.it.verbalizers.time import TimeFst class VerbalizeFst(GraphFst): @@ -44,7 +46,18 @@ def __init__(self, deterministic: bool = True): measure_graph = measure.fst money = MoneyFst(decimal=decimal, deterministic=deterministic) money_graph = money.fst + cardinal_tagger = CardinalTagger(deterministic=deterministic) + time = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic) + time_graph = time.fst - graph = cardinal_graph | decimal_graph | electronic_graph | whitelist_graph | measure_graph | money_graph + graph = ( + cardinal_graph + | decimal_graph + | electronic_graph + | whitelist_graph + | measure_graph + | money_graph + | time_graph + ) self.fst = graph diff --git a/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py index 7f00e5bd8..d92c79848 100644 --- a/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/it/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -24,7 +25,7 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.it.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): @@ -47,7 +48,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"it_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst @@ -67,4 +68,3 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ja/__init__.py b/nemo_text_processing/text_normalization/ja/__init__.py new file mode 100644 index 000000000..0928420fd --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/ja/data/__init__.py b/nemo_text_processing/text_normalization/ja/data/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/date/__init__.py b/nemo_text_processing/text_normalization/ja/data/date/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/date/day.tsv b/nemo_text_processing/text_normalization/ja/data/date/day.tsv new file mode 100644 index 000000000..09258cb4c --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/date/day.tsv @@ -0,0 +1,31 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/date/era.tsv b/nemo_text_processing/text_normalization/ja/data/date/era.tsv new file mode 100644 index 000000000..b932b7fb3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/date/era.tsv @@ -0,0 +1,12 @@ +令和 +平成 +昭和 +大正 +明治 +西暦 +和暦 +西洋暦 +グレゴリオ暦 +紀元前 +紀元 +紀元後 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv b/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv new file mode 100644 index 000000000..1c95675c2 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv @@ -0,0 +1,5 @@ +R. 令和 +H. 平成 +S. 昭和 +T. 大正 +M. 明治 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/date/month.tsv b/nemo_text_processing/text_normalization/ja/data/date/month.tsv new file mode 100644 index 000000000..f992b4d28 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/date/month.tsv @@ -0,0 +1,12 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/date/week.tsv b/nemo_text_processing/text_normalization/ja/data/date/week.tsv new file mode 100644 index 000000000..41556b4db --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/date/week.tsv @@ -0,0 +1,15 @@ +月 月曜日 +火 火曜日 +水 水曜日 +木 木曜日 +金 金曜日 +土 土曜日 +日 日曜日 +祝 祝日 +月曜日 +火曜日 +水曜日 +木曜日 +金曜日 +土曜日 +日曜日 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py b/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv new file mode 100644 index 000000000..ede6b97b7 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv new file mode 100644 index 000000000..1585fe836 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv new file mode 100644 index 000000000..562e63265 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +2 二十 +3 三十 +4 四十 +5 五十 +6 六十 +7 七十 +8 八十 +9 九十 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv new file mode 100644 index 000000000..7fab21718 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 零 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/symbol.tsv b/nemo_text_processing/text_normalization/ja/data/symbol.tsv new file mode 100644 index 000000000..67ad785f0 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/symbol.tsv @@ -0,0 +1,23 @@ +& アンド +# ハッシュタグ +@ アット +§ セクション +™ トレードマーク +® 登録商標マーク +© 著作権 +_ アンダースコア +% パーセント +* 星印 ++ プラス +/ スラッシュ += エコール +^ 曲折アクセント記号 +| 縦棒 +~ ティルダ +$ ドール +£ ポンド +€ ユーロ +₩ ウォン +¥ 円 +° 度 +º 度 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/time/__init__.py b/nemo_text_processing/text_normalization/ja/data/time/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/time/division.tsv b/nemo_text_processing/text_normalization/ja/data/time/division.tsv new file mode 100644 index 000000000..f6beedce8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/time/division.tsv @@ -0,0 +1,23 @@ +今朝 +今夜 +今晩 +午前 +午後 +夕方 +夜中 +夜 +夜半 +朝 +早朝 +明け方 +昼 +深夜 +毎朝 +毎夜 +毎晩 +毎日 +真夜中 +翌日 +未明 +正午 +真夜中の \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/time/hour.tsv b/nemo_text_processing/text_normalization/ja/data/time/hour.tsv new file mode 100644 index 000000000..1d5c08425 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/time/hour.tsv @@ -0,0 +1,24 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/time/minute.tsv b/nemo_text_processing/text_normalization/ja/data/time/minute.tsv new file mode 100644 index 000000000..5e8276a00 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/time/minute.tsv @@ -0,0 +1,60 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十二 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/time/second.tsv b/nemo_text_processing/text_normalization/ja/data/time/second.tsv new file mode 100644 index 000000000..5e8276a00 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/time/second.tsv @@ -0,0 +1,60 @@ +1 一 +2 二 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 +20 二十 +21 二十一 +22 二十二 +23 二十三 +24 二十四 +25 二十五 +26 二十六 +27 二十七 +28 二十八 +29 二十九 +30 三十 +31 三十一 +32 三十二 +33 三十三 +34 三十四 +35 三十五 +36 三十六 +37 三十七 +38 三十八 +39 三十九 +40 四十 +41 四十一 +42 四十二 +43 四十三 +44 四十四 +45 四十五 +46 四十六 +47 四十七 +48 四十八 +49 四十九 +50 五十 +51 五十一 +52 五十二 +53 五十三 +54 五十四 +55 五十五 +56 五十六 +57 五十七 +58 五十八 +59 五十九 +60 六十 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/whitelist.tsv b/nemo_text_processing/text_normalization/ja/data/whitelist.tsv new file mode 100644 index 000000000..d0d7bef70 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/whitelist.tsv @@ -0,0 +1,40 @@ +Dr. ドクター +dr. ドクター +Mr. ミスター +mr. ミスター +Ms. ミス +ms. ミス +Mrs. ミシーズ +mrs. ミシーず +st. ストリード +St. ストリード +mt. マウント +Mt. マウント +Prof. プロフェッサー +prof. プロフェッサー +sr. シニア +Sr. シニア +jr. ジュニア +Jr. ジュニア +rd. ロード +Rd. ロード +Ave. アベニュー +ave. アベニュー +no. ナンバー +No. ナンバー +( カッコ +) カッコ閉じる ++ プラス +- マイナス +Σ シグマ +η エタ +κ カッパ +ω オメガ +σ シグマ +α アルフ +ν ヌー +δ デルタ +ι イオタ +vs. バーサス +Ph.D. p h d +etc. エトセトラ diff --git a/nemo_text_processing/text_normalization/ja/graph_utils.py b/nemo_text_processing/text_normalization/ja/graph_utils.py new file mode 100644 index 000000000..f4c30b6ee --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/graph_utils.py @@ -0,0 +1,297 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_NARROW_NON_BREAK_SPACE = u"\u202f" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() +NEMO_SPACES_AND_ALHPANUMERICS = pynini.closure( + NEMO_SIGMA + | pynini.closure(NEMO_SPACE) + | pynini.closure(NEMO_NON_BREAKING_SPACE) + | pynini.closure(NEMO_DIGIT) + | pynini.closure(NEMO_WHITE_SPACE) +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z" +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: 'pynini.FstLike', original_graph_weight: float = None, capitalized_graph_weight: float = None +) -> 'pynini.FstLike': + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/ja/taggers/__init__.py b/nemo_text_processing/text_normalization/ja/taggers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/taggers/cardinal.py b/nemo_text_processing/text_normalization/ja/taggers/cardinal.py new file mode 100644 index 000000000..ff80f6a3b --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/cardinal.py @@ -0,0 +1,191 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 23 -> cardinal { integer: "二十三" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + + no_one = pynini.difference(NEMO_DIGIT, "1") + no_zero_and_one = pynini.difference(no_one, "0") + + graph_digit_alt = no_zero_and_one @ graph_digit + graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + + graph_all = (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen | graph_digit + + hundreds = NEMO_DIGIT**3 + graph_hundred_component = (pynini.cross('1', '百') | (graph_digit_alt + pynutil.insert('百'))) + pynini.union( + pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_all) + ) + graph_hundred = hundreds @ graph_hundred_component + + thousands = NEMO_DIGIT**4 + graph_thousand_component = (pynini.cross('1', '千') | (graph_digit_alt + pynutil.insert('千'))) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_thousand_component_alt = (graph_digit + pynutil.insert('千')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + # this grammar is for larger number in later gramamr + graph_thousand = thousands @ graph_thousand_component + + ten_thousands = NEMO_DIGIT**5 + graph_ten_thousand_component = (graph_digit + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_thousand = ten_thousands @ graph_ten_thousand_component + self.man = graph_ten_thousand.optimize() + + hundred_thousands = NEMO_DIGIT**6 + hundred_thousands_position = NEMO_DIGIT**2 + hundred_thousands_position = hundred_thousands_position @ graph_all + graph_hundred_thousand_component = (hundred_thousands_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component + + millions = NEMO_DIGIT**7 + million_position = NEMO_DIGIT**3 + million_position = million_position @ graph_hundred_component + graph_million_component = (million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_million = millions @ graph_million_component + + ten_millions = NEMO_DIGIT**8 + ten_million_position = NEMO_DIGIT**4 + ten_million_position = ten_million_position @ graph_thousand_component_alt + graph_ten_million_component = (ten_million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_million = ten_millions @ graph_ten_million_component + + hundred_millions = NEMO_DIGIT**9 + graph_hundred_million_component = (graph_digit + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_million = hundred_millions @ graph_hundred_million_component + + thousand_millions = NEMO_DIGIT**10 + thousand_millions_position = NEMO_DIGIT**2 + thousand_millions_position = thousand_millions_position @ graph_all + graph_thousand_million_component = (thousand_millions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_thousand_million = thousand_millions @ graph_thousand_million_component + + ten_billions = NEMO_DIGIT**11 + ten_billions_position = NEMO_DIGIT**3 + ten_billions_position = ten_billions_position @ graph_hundred_component + graph_ten_billions_component = (ten_billions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_ten_billions = ten_billions @ graph_ten_billions_component + + hundred_billions = NEMO_DIGIT**12 + hundred_billions_position = NEMO_DIGIT**4 + hundred_billions_position = hundred_billions_position @ graph_thousand_component_alt + graph_hundred_billions_component = (hundred_billions_position + pynutil.insert('億')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_all), + ) + graph_hundred_billions = hundred_billions @ graph_hundred_billions_component + + graph = pynini.union( + graph_hundred_billions, + graph_ten_billions, + graph_thousand_million, + graph_hundred_million, + graph_ten_million, + graph_million, + graph_hundred_thousand, + graph_ten_thousand, + graph_thousand, + graph_hundred, + graph_all, + graph_zero, + ) + self.just_cardinals = graph.optimize() + + optional_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("マイナス", "-")) + pynutil.insert("\"") + ) + + final_graph = ( + optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/date.py b/nemo_text_processing/text_normalization/ja/taggers/date.py new file mode 100644 index 000000000..a8a469252 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/date.py @@ -0,0 +1,227 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NARROW_NON_BREAK_SPACE, + NEMO_NON_BREAKING_SPACE, + GraphFst, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + 2024/01/30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024/1/30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024-01-30 -> date { year: "二千二十四" month: "一" day: "三十" } + 2024.01.30 -> date { year: "二千二十四" month: "一" day: "三十" } + H.6 -> date { era: "平成" yera: "六年" } + R.1 -> date { era: "令和" "year: "元年" } + S.5 -> date { era: "昭和" "year: "五年" } + T.5 -> date { era: "大正" "year: "五年" } + M.5 -> date { era: "明治" "year: "五年" } + 21日月曜日 -> tokens { date { day: "二十一日" weekday: "月曜日" } } + 70年代 -> date { year: "七十年代" } + 西暦794年 -> tokens { date { era: "西暦" year: "七百九十四年" } } + 1月1日(月)〜3日(水) + -> tokens { date { month: "一月" day: "一日" weekday: "月曜日" } } tokens { name: "から" } tokens { date { day: "三日" weekday: "水曜日" } } + 70〜80年代 + -> tokens { cardinal { integer: "七十" } } tokens { name: "から" } tokens { date { year: "八十年代" } } + 7月5〜9日(月〜金) + -> tokens { date { month: "七月" } } tokens { cardinal { integer: "五" } } tokens { name: "から" } tokens { date { day: "九日" weekday: "月曜日" } } tokens { name: "から" } tokens { date { weekday: "金曜日" } } + 7月初旬〜9月中旬 + -> tokens { date { month: "七月" } } tokens { name: "初" } tokens { name: "旬" } tokens { name: "から" } tokens { date { month: "九月" } } tokens { name: "中" } tokens { name: "旬" } + 3〜4月 + -> tokens { cardinal { integer: "三" } } tokens { name: "から" } tokens { date { month: "四月" } } + 2023年3月1日(水)〜6月12日(火) + -> tokens { date { year: "二千二十三年" month: "三月" day: "一日" weekday: "水曜日" } } tokens { name: "から" } tokens { date { month: "六月" day: "十二日" weekday: "火曜日" } } + 10月中旬〜11月上旬 + -> tokens { date { month: "十月" } } tokens { date { month: "中旬" } } tokens { name: "から" } tokens { date { month: "十一月" } } tokens { date { month: "上旬" } } + 1976年7月17日〜8月1日 + -> tokens { date { year: "千九百七十六年" month: "七月" day: "十七日" } } tokens { name: "から" } tokens { date { month: "八月" day: "一日" } } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + + month = pynutil.delete("0").ques + pynini.string_file(get_abs_path("data/date/month.tsv")) + day = pynutil.delete("0").ques + pynini.string_file(get_abs_path("data/date/day.tsv")) + week = pynini.string_file(get_abs_path("data/date/week.tsv")) + era = pynini.string_file(get_abs_path("data/date/era.tsv")) + era_abbrev = pynini.string_file(get_abs_path("data/date/era_abbrev.tsv")) + + signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") + delete_spaces = pynini.closure( + pynutil.delete(" ") | pynutil.delete(NEMO_NARROW_NON_BREAK_SPACE) | pynutil.delete(NEMO_NON_BREAKING_SPACE) + ) + + era_component = pynutil.insert("era: \"") + era + pynutil.insert("\"") + era_abbrev_component = pynutil.insert("era: \"") + era_abbrev + pynutil.insert("\"") + year_component = pynutil.insert("year: \"") + graph_cardinal + pynutil.insert("年") + pynutil.insert("\"") + month_component = pynutil.insert("month: \"") + month + pynutil.insert("月") + pynutil.insert("\"") + day_component = pynutil.insert("day: \"") + day + pynutil.insert("日") + pynutil.insert("\"") + + front_bracket = ( + ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("(") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("（") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("（") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + ) + preceding_bracket = ( + ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete(")") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("）") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + | ( + pynini.closure(pynutil.delete(delete_spaces)) + + pynutil.delete("）") + + pynini.closure(pynutil.delete(delete_spaces)) + ) + ) + # this graph optionally accepts () around weekday to accomodate to inputs like (月〜金), thus being longer + + week_component = ( + (front_bracket + pynutil.insert("weekday: \"") + week + preceding_bracket + pynutil.insert("\"")) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynini.cross("〜", "から") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynutil.delete("・") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + ) + + # era, year, month, date + graph_basic_date = ( # (R.|令和)2024/01/01, (R.|令和)2024/01/01/(水), (R.|令和)01/01, (R.|令和)01/01(水) + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynini.closure(year_component + signs + pynutil.insert(" "), 1) + + month_component + + signs + + pynutil.insert(" ") + + day_component + + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + ) + + # 2024年, 9月, 28日 + individual_year_component = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + pynini.accep("年") + + pynutil.insert("\"") + ) + # this extra individual year component is to accomodate inputs R. 2024 with out "年" + # the inputs may or maynot include "年", thus below: + individual_year_component_2 = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + (pynini.accep("世紀") | pynini.accep("")) + + pynutil.insert("\"") + ) | ( + era_abbrev_component + + pynutil.insert(" ") + + pynutil.insert("year: \"") + + graph_cardinal + + pynutil.insert("年") + + pynutil.insert("\"") + ) + + individual_month_component = ( + pynutil.insert("month: \"") + month + pynini.accep("月") + pynutil.insert("\"") + ) | ( + pynutil.insert("month: \"") + + (pynini.accep("中旬") | pynini.accep("下旬") | pynini.accep("上旬")) + + pynutil.insert("\"") + ) + individual_day_component = ( + pynutil.insert("day: \"") + graph_cardinal + pynini.accep("日") + pynutil.insert("\"") + ) + + graph_individual_component = ( + individual_year_component + | individual_month_component + | individual_day_component + | week_component + | individual_year_component_2 + ) + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + + # combined the above individual date components + graph_individual_component_combined = ( + (individual_year_component + pynutil.insert(" ") + individual_month_component) + | (individual_month_component + pynutil.insert(" ") + individual_day_component) + | ( + individual_year_component + + pynutil.insert(" ") + + individual_month_component + + pynutil.insert(" ") + + individual_day_component + ) + ) + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) + + nendai = pynini.accep("年代") + era_nendai = ( + pynini.closure(era_component + pynutil.insert(" "), 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + nendai + + pynutil.insert("\"") + ) + + graph_all_date = ( + graph_basic_date | graph_individual_component | graph_individual_component_combined | era_nendai + ) + + final_graph = graph_all_date + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/decimal.py b/nemo_text_processing/text_normalization/ja/taggers/decimal.py new file mode 100644 index 000000000..8fdea4c87 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal, e.g. + 0.5 -> decimal { integer_part: "零" fractional_part: "五" } + -0.5万 -> decimal { negative: "マイナス" integer_part: "零" fractional_part: "五" quantity: "万"} + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_before_decimal = cardinal.just_cardinals + cardinal_after_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + graph_integer = pynutil.insert('integer_part: \"') + cardinal_before_decimal + pynutil.insert("\"") + graph_fraction = ( + pynutil.insert("fractional_part: \"") + + pynini.closure((cardinal_after_decimal | zero), 1) + + pynutil.insert("\"") + ) + graph_decimal_no_sign = graph_integer + pynutil.delete('.') + pynutil.insert(" ") + graph_fraction + + graph_optional_sign = ( + pynutil.insert("negative: \"") + + (pynini.cross("-", "マイナス") | pynini.accep("マイナス")) + + pynutil.insert("\"") + ) + + graph_decimal = graph_decimal_no_sign | (graph_optional_sign + pynutil.insert(" ") + graph_decimal_no_sign) + + self.just_decimal = graph_decimal_no_sign.optimize() + + final_graph = self.add_tokens(graph_decimal) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/fraction.py b/nemo_text_processing/text_normalization/ja/taggers/fraction.py new file mode 100644 index 000000000..94fb4af68 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/fraction.py @@ -0,0 +1,125 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying fractions, e.g. + 1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } + 1と3/4 -> fraction { integer: "一" denominator: "四" numerator: "三" } + 一荷四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + ルート三分の一 -> fraction { denominator: "√3" numerator: "1" } + 一点六五分の五十 -> fraction { denominator: "1.65" numerator: "50" } + マイナス1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinal = cardinal.just_cardinals + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + slash = pynutil.delete('/') + morphemes = pynini.accep('分の') + root = pynini.accep('√') + + decimal_number = ( + cardinal + + pynini.cross(".", "点") + + pynini.closure(pynini.closure(graph_digit) | pynini.closure(graph_zero)) + ) + + integer_component = ( + pynutil.insert('integer_part: \"') + + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + pynutil.insert("\"") + ) + integer_component_with_char = ( + pynutil.insert('integer_part: \"') + + ( + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + (pynini.accep("と") | pynini.accep("荷")) + ) + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + ) + denominator_component = ( + pynutil.insert("denominator: \"") + + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + pynutil.insert("\"") + ) + numerator_component = ( + pynutil.insert("numerator: \"") + + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + + pynutil.insert("\"") + ) + + # 3/4, 1 3/4, 1と3/4, -3/4, -1 3/4, 1と3/4, √1と3/4 and any combination of root number, cardinal number and decimal number + graph_fraction_slash = ( + pynini.closure( + (integer_component + pynini.accep(NEMO_SPACE)) + | (integer_component_with_char + pynutil.insert(NEMO_SPACE)), + 0, + 1, + ) + + numerator_component + + slash + + pynutil.insert(NEMO_SPACE) + + denominator_component + ) + + # 4分の3 -4分の3 and any combs + graph_fraction_word = ( + pynini.closure( + ( + integer_component + pynini.accep(NEMO_SPACE) + | integer_component_with_char + pynutil.insert(NEMO_SPACE) + ), + 0, + 1, + ) + + denominator_component + + pynutil.insert(NEMO_SPACE) + + pynutil.insert("morphosyntactic_features: \"") + + morphemes + + pynutil.insert("\"") + + pynutil.insert(NEMO_SPACE) + + numerator_component + ) + + optional_sign = ( + pynutil.insert("negative: \"") + + (pynini.accep("マイナス") | pynini.cross("-", "マイナス")) + + pynutil.insert("\"") + ) + + graph_fraction_slash_sigh = pynini.closure(optional_sign + pynutil.insert(NEMO_SPACE), 0, 1) + ( + graph_fraction_slash | graph_fraction_word + ) + + graph = graph_fraction_slash_sigh # | + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/ordinal.py b/nemo_text_processing/text_normalization/ja/taggers/ordinal.py new file mode 100644 index 000000000..d88608e01 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying ordinal, e.g. + 第100 -> ordinal { integer: "第百" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + morpheme_pre = pynini.accep('第') + morpheme_post = pynini.accep('番目') + graph_ordinal = pynini.union(morpheme_pre + graph_cardinal, graph_cardinal + morpheme_post) + + final_graph = pynutil.insert("integer: \"") + graph_ordinal + pynutil.insert("\"") + + graph_ordinal_final = self.add_tokens(final_graph) + self.fst = graph_ordinal_final.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/punctuation.py b/nemo_text_processing/text_normalization/ja/taggers/punctuation.py new file mode 100644 index 000000000..c5df8388c --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/punctuation.py @@ -0,0 +1,70 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#$%&'()*+,-./:;<=>?@^_`{|}。，；：《》“”·~【】！？、‘’.<>-——_、。.「」『』‘`／・；’”“”‷･〔〕々〃ゝゞヽ〲〱〳〴〵ヾ〆，~" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + whitelist_symbols = load_labels(get_abs_path("data/symbol.tsv")) + whitelist_symbols = [x[0] for x in whitelist_symbols] + self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + ( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)) + | (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)) + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + range_component = pynini.cross("〜", "から") | pynini.accep( + "から" + ) # forcing this conversion for special tilde + + self.graph = punct | pynutil.add_weight(range_component, -1.0) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/time.py b/nemo_text_processing/text_normalization/ja/taggers/time.py new file mode 100644 index 000000000..7c74bc53e --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/time.py @@ -0,0 +1,112 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 1時30分 -> time { hours: "一" minutes: "三十" } + 今夜0時 -> time { suffix: "今夜" hours: "零" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + + hour_clock = pynini.string_file(get_abs_path("data/time/hour.tsv")) + minute_clock = pynini.string_file(get_abs_path("data/time/minute.tsv")) + second_clock = pynini.string_file(get_abs_path("data/time/second.tsv")) + division = pynini.string_file(get_abs_path("data/time/division.tsv")) + + division_component = pynutil.insert("suffix: \"") + division + pynutil.insert("\"") + hour_component = ( + pynutil.insert("hours: \"") + + (graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) + + (pynini.accep("時") | pynini.accep("時間") | pynini.accep("時頃")) + + pynutil.insert("\"") + ) + minute_component = pynutil.insert("minutes: \"") + ( + graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal) + ) + pynini.accep("分") + pynini.closure((pynini.accep("過ぎ") | pynini.accep("頃")), 0, 1) + pynutil.insert( + "\"" + ) | ( + pynutil.insert("minutes: \"") + + pynini.accep("半") + + pynini.closure((pynini.accep("過ぎ") | pynini.accep("頃")), 0, 1) + + pynutil.insert("\"") + ) + second_component = ( + pynutil.insert("seconds: \"") + + (graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) + + pynini.accep("秒") + + pynutil.insert("\"") + ) + + graph_individual_time = pynini.closure(division_component + pynutil.insert(" "), 0, 1) + ( + hour_component + | minute_component + | second_component + | (hour_component + pynutil.insert(" ") + minute_component) + | (hour_component + pynutil.insert(" ") + minute_component + pynutil.insert(" ") + second_component) + | (minute_component + pynutil.insert(" ") + second_component) + ) + + colon = pynutil.delete(":") + hour_clock_component = ( + pynutil.insert("hours: \"") + + pynutil.delete("0").ques + + hour_clock + + pynutil.insert("時") + + pynutil.insert("\"") + ) + minute_clock_component = ( + pynutil.insert("minutes: \"") + + pynutil.delete("0").ques + + minute_clock + + pynutil.insert("分") + + pynutil.insert("\"") + ) + second_clock_component = ( + pynutil.insert("seconds: \"") + + pynutil.delete("0").ques + + second_clock + + pynutil.insert("秒") + + pynutil.insert("\"") + ) + + graph_clock = ( + hour_clock_component + + pynutil.insert(" ") + + colon + + minute_clock_component + + pynutil.insert(" ") + + colon + + second_clock_component + ) | (hour_clock_component + pynutil.insert(" ") + colon + minute_clock_component) + + graph = graph_individual_time | graph_clock + + graph_final = self.add_tokens(graph) + self.fst = graph_final.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..f992e9b70 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py @@ -0,0 +1,94 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, generator_main +from nemo_text_processing.text_normalization.ja.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ja.taggers.date import DateFst +from nemo_text_processing.text_normalization.ja.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ja.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.ja.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ja.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.ja.taggers.time import TimeFst +from nemo_text_processing.text_normalization.ja.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.ja.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join(cache_dir, f"zh_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + else: + cardinal = CardinalFst(deterministic=deterministic) + date = DateFst(cardinal=cardinal, deterministic=deterministic) + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + time = TimeFst(cardinal=cardinal, deterministic=deterministic) + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + punctuation = PunctuationFst(deterministic=deterministic) + + classify = pynini.union( + pynutil.add_weight(date.fst, 1.1), + pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(time.fst, 1.1), + pynutil.add_weight(whitelist.fst, 1.1), + pynutil.add_weight(cardinal.fst, 1.1), + pynutil.add_weight(decimal.fst, 3.05), + pynutil.add_weight(ordinal.fst, 1.1), + pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(word.fst, 100), + ) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") + tagger = pynini.closure(token, 1) + + self.fst = tagger + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/nemo_text_processing/text_normalization/ja/taggers/whitelist.py b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py new file mode 100644 index 000000000..2f9391ddf --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py @@ -0,0 +1,40 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelisted tokens + e.g. Dr. -> tokens { name: "ドクター" } + Ms. -> { name: "ミス" } + jr. -> { name: "ジュニア" } + etc. -> { name: "エトセトラ" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + graph = (pynutil.insert('name: "')) + (whitelist) + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/word.py b/nemo_text_processing/text_normalization/ja/taggers/word.py new file mode 100644 index 000000000..b1403221b --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. 文字 -> tokens { name: "文字" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/ja/utils.py b/nemo_text_processing/text_normalization/ja/utils.py new file mode 100644 index 000000000..65523afed --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/utils.py @@ -0,0 +1,56 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +# def load_labels(abs_path): +# """ +# loads relative path file as dictionary + +# Args: +# abs_path: absolute path + + +# Returns dictionary of mappings +# """ +# #label_tsv = open(abs_path, encoding="utf-8") +# label_tsv = open(abs_path, "r") +# labels = list(csv.reader(label_tsv, delimiter="\t")) +# return labels +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/__init__.py b/nemo_text_processing/text_normalization/ja/verbalizers/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/ja/verbalizers/cardinal.py new file mode 100644 index 000000000..cc9b58675 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/cardinal.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinals, e.g. + cardinal { integer: "一" } -> 一 + cardinal { negative: "-" integer: "二十三" } -> マイナス二十三 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + optional_sign = ( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + delete_space + ) + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + + final_graph = pynini.closure(optional_sign, 0, 1) + graph + + final_graph = self.delete_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/date.py b/nemo_text_processing/text_normalization/ja/verbalizers/date.py new file mode 100644 index 000000000..8292c622a --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/date.py @@ -0,0 +1,55 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing date e.g. + date { year: "二千二十四" month: "三" day: "四" } -> 二千二十四年三月四日 + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + era_component = pynutil.delete("era: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + year_component = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + month_component = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + day_component = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + week_component = ( + pynutil.delete("weekday: \"") + + pynini.closure(delete_space) + + pynini.closure(NEMO_NOT_QUOTE) + + pynini.closure(delete_space) + + pynutil.delete("\"") + ) + + graph_basic_date = ( + pynini.closure(era_component + delete_space, 0, 1) + + pynini.closure(year_component + delete_space, 0, 1) + + pynini.closure(month_component + delete_space, 0, 1) + + pynini.closure(day_component, 0, 1) + + pynini.closure((delete_space + week_component) | (week_component), 0, 1) + ) | month_component + delete_space + week_component + + final_graph = graph_basic_date + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py b/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py new file mode 100644 index 000000000..f4200001c --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing decimal, e.g. + decimal { integer_part: "零" fractional_part: "五" } -> 零点五 + decimal { positive: "-" integer_part: "零" fractional_part: "五" } -> マイナス零点五 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + graph_integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + graph_fraction = ( + pynutil.delete("fractional_part: \"") + + pynutil.insert("点") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph_optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + graph_decimal_no_sign = graph_integer + pynutil.delete(" ") + graph_fraction + graph_decimal = graph_decimal_no_sign | (graph_optional_sign + pynutil.delete(" ") + graph_decimal_no_sign) + + delete_tokens = self.delete_tokens(graph_decimal) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py new file mode 100644 index 000000000..4743d3bcd --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py @@ -0,0 +1,123 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing fractionss, e.g. + tokens { fraction { denominator: "二" numerator: "一"} } -> 1/2 + tokens { fraction { integer: "一" denominator: "四" numerator: "三" } } -> 1と3/4 + tokens { fraction { integer: "1" denominator: "4" numerator: "3" } } -> 一荷四分の三 + tokens { fraction { denominator: "√3" numerator: "1" } } -> ルート三分の一 + tokens { fraction { denominator: "1.65" numerator: "50" } } -> 一点六五分の五十 + tokens { fraction { denominator: "二" numerator: "一"} } -> マイナス1/2 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + denominator_component = ( + pynutil.delete('denominator: \"') + pynini.closure(NEMO_NOT_QUOTE - "√") + pynutil.delete("\"") + ) + numerator_component = ( + pynutil.delete('numerator: \"') + pynini.closure(NEMO_NOT_QUOTE - "√") + pynutil.delete("\"") + ) + + # 1/3 + graph_regular_fraction = ( + denominator_component + pynutil.delete(NEMO_SPACE) + pynutil.insert("分の") + numerator_component + ) + + denominator_component_root = ( + pynutil.delete('denominator: \"') + + pynini.cross("√", "ルート") + + pynini.closure(NEMO_NOT_QUOTE - "√") + + pynutil.delete("\"") + ) + numerator_component_root = ( + pynutil.delete('numerator: \"') + + pynini.cross("√", "ルート") + + pynini.closure(NEMO_NOT_QUOTE - "√") + + pynutil.delete("\"") + ) + # √3/1 + graph_regular_fraction_root = ( + (denominator_component_root | denominator_component) + + pynutil.delete(NEMO_SPACE) + + pynutil.insert("分の") + + (numerator_component_root | numerator_component) + ) + + # 3分の1 + graph_regular_fraction_char = ( + (denominator_component | denominator_component_root) + + pynutil.delete(NEMO_SPACE) + + pynutil.delete("morphosyntactic_features: \"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + pynutil.delete(NEMO_SPACE) + + (numerator_component | numerator_component_root) + ) + + graph_integer = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(pynini.cross("√", "ルート"), 0, 1) + + pynini.closure( + NEMO_NOT_QUOTE - pynini.union("荷", "と", "√") + ) # had to remove these 3 items fron nemo_not _quote so the root is properly converted in a deterministic way. + + pynutil.insert("荷") + + pynutil.delete("\"") + ) + + graph_integer_with_char = ( + pynutil.delete("integer_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(pynini.cross("√", "ルート"), 0, 1) + + pynini.closure(NEMO_NOT_QUOTE - pynini.union("荷", "と", "√")) + + (pynini.accep("と") | pynini.accep("荷")) + + pynutil.delete("\"") + ) + + graph_regular_integer = ( + (graph_integer | graph_integer_with_char) + + delete_space + + (graph_regular_fraction | graph_regular_fraction_root | graph_regular_fraction_char) + ) + + optional_sign = ( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + + delete_space + ) + + graph = pynini.closure(optional_sign, 0, 1) + ( + graph_regular_integer | graph_regular_fraction | graph_regular_fraction_root | graph_regular_fraction_char + ) + + # graph = pynini.closure(graph_optional_sign, 0, 1) + graph_fractions + + final_graph = self.delete_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/ja/verbalizers/ordinal.py new file mode 100644 index 000000000..e0a34d634 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing ordinal e.g. + tokens { ordinal { integer: "第一千万" } } -> 第一千万 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_integer = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + + final_graph = graph_integer + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py new file mode 100644 index 000000000..8b196dcaf --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_SPACE, + NEMO_SIGMA, + delete_space, + generator_main, +) +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "zh_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def set_punct_dict(self): + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '՚', + '՝', + 'י', + '׳', + 'ߴ', + 'ߵ', + 'ᑊ', + 'ᛌ', + '᾽', + '᾿', + '`', + '´', + '῾', + '‘', + '’', + '‛', + '′', + '‵', + 'ꞌ', + '＇', + '｀', + '𖽑', + '𖽒', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + """ + + remove_space_around_single_quote = pynini.cdrewrite( + delete_space, NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) + ) + # this works if spaces in between (good) + # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA + + graph = remove_space_around_single_quote.optimize() + + return graph diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py new file mode 100644 index 000000000..3ff05fa57 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py @@ -0,0 +1,81 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil, utf8 + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_PUNCT, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class PostProcessor(GraphFst): + ''' + Postprocessing of TN, now contains: + 1. punctuation removal + 2. letter case conversion + 3. oov tagger + ''' + + def __init__( + self, + remove_puncts: bool = False, + to_upper: bool = False, + to_lower: bool = False, + tag_oov: bool = False, + ): + super().__init__(name="PostProcessor", kind="processor") + + graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + if remove_puncts: + remove_puncts_graph = pynutil.delete( + pynini.union(NEMO_PUNCT, pynini.string_file(get_abs_path('data/char/punctuations_zh.tsv'))) + ) + graph @= pynini.cdrewrite(remove_puncts_graph, "", "", NEMO_SIGMA).optimize() + + if to_upper or to_lower: + if to_upper: + conv_cases_graph = pynini.inverse(pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv'))) + else: + conv_cases_graph = pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv')) + + graph @= pynini.cdrewrite(conv_cases_graph, "", "", NEMO_SIGMA).optimize() + + if tag_oov: + zh_charset_std = pynini.string_file(get_abs_path("data/char/charset_national_standard_2013_8105.tsv")) + zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv")) + + zh_charset = ( + zh_charset_std | zh_charset_ext | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv")) + ) + en_charset = NEMO_DIGIT | NEMO_ALPHA | NEMO_PUNCT | NEMO_WHITE_SPACE + charset = zh_charset | en_charset + + with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f: + tags = f.readline().strip().split('\t') + assert len(tags) == 2 + ltag, rtag = tags + + oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset) + tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag) + graph @= pynini.cdrewrite(tag_oov_graph, "", "", NEMO_SIGMA).optimize() + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/time.py b/nemo_text_processing/text_normalization/ja/verbalizers/time.py new file mode 100644 index 000000000..7058f437b --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/time.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time e.g. + + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + hour_component = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + minute_component = pynutil.delete("minutes: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + second_component = pynutil.delete("seconds: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + division_component = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_basic_time = pynini.closure(division_component + pynutil.delete(" "), 0, 1) + ( + (hour_component + pynutil.delete(" ") + minute_component + pynutil.delete(" ") + second_component) + | (hour_component + pynutil.delete(" ") + minute_component) + | hour_component + | minute_component + | second_component + ) + + final_graph = graph_basic_time + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py new file mode 100644 index 000000000..6a16f96d9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py @@ -0,0 +1,66 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ja.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.ja.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ja.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.ja.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ja.verbalizers.time import TimeFst +from nemo_text_processing.text_normalization.ja.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.ja.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + date = DateFst(deterministic=deterministic) + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + decimal = DecimalFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + fraction = FractionFst(deterministic=deterministic) + + # money = MoneyFst(decimal=decimal, deterministic=deterministic) + # measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + + graph = pynini.union( + date.fst, + cardinal.fst, + ordinal.fst, + decimal.fst, + fraction.fst, + word.fst, + time.fst, + whitelist.fst, + ) + graph = pynini.closure(delete_space) + graph + pynini.closure(delete_space) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py new file mode 100644 index 000000000..750598649 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py @@ -0,0 +1,54 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.verbalizers.postprocessor import PostProcessor +from nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst + +# from nemo.utils import logging + + +class VerbalizeFinalFst(GraphFst): + """ """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"jp_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + else: + token_graph = VerbalizeFst(deterministic=deterministic) + + token_verbalizer = ( + pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") + ) + verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space) + + postprocessor = PostProcessor( + remove_puncts=False, + to_upper=False, + to_lower=False, + tag_oov=False, + ) + + self.fst = (verbalizer @ postprocessor.fst).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py new file mode 100644 index 000000000..11b0b3ae0 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist + e.g. tokens { name: "ミスター" } } -> ミスター + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/word.py b/nemo_text_processing/text_normalization/ja/verbalizers/word.py new file mode 100644 index 000000000..afd5d1037 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WordFst(GraphFst): + ''' + tokens { char: "文字" } -> 文字 + ''' + + def __init__(self, deterministic: bool = True): + super().__init__(name="char", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("name: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"") + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/__init__.py b/nemo_text_processing/text_normalization/ko/__init__.py new file mode 100644 index 000000000..dd0e509b3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/ko/data/__init__.py b/nemo_text_processing/text_normalization/ko/data/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/date/__init__.py b/nemo_text_processing/text_normalization/ko/data/date/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv b/nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv new file mode 100644 index 000000000..2f54cee92 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/date/exceptions.tsv @@ -0,0 +1,2 @@ +6 유 +10 시 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/date/week.tsv b/nemo_text_processing/text_normalization/ko/data/date/week.tsv new file mode 100644 index 000000000..bc205bc3f --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/date/week.tsv @@ -0,0 +1,8 @@ +월 월요일 +화 화요일 +수 수요일 +목 목요일 +금 금요일 +토 토요일 +일 일요일 +공 공휴일 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/electronic/__init__.py b/nemo_text_processing/text_normalization/ko/data/electronic/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv b/nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv new file mode 100644 index 000000000..f562cfbad --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/electronic/cc_cues.tsv @@ -0,0 +1,11 @@ +카드 끝자리 카드 끝자리 +카드 마지막 네자리 카드 마지막 네자리 +카드 마지막 4자리 카드 마지막 네자리 +신용카드 번호 신용카드 번호 +신용카드 신용카드 +체크카드 번호 체크카드 번호 +체크카드 체크카드 +카드번호 카드번호 +결제 카드 결제 카드 +결제카드 결제카드 +카드 카드 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv b/nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv new file mode 100644 index 000000000..3d04ca298 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/electronic/domain.tsv @@ -0,0 +1,28 @@ +.com 닷컴 +.org 닷 오알지 +.gov 닷 거브 +.edu 닷 에듀 +.net 닷 넷 +.ai 닷 에이아이 +.io 닷 아이오 +.dev 닷 데브 +.app 닷 앱 +.cloud 닷 클라우드 +.shop 닷 샵 +.store 닷 스토어 +.co 닷 씨오 +.me 닷 미 +.kr 닷 케이알 +.co.kr 닷 씨오 닷 케이알 +.ac.kr 닷 에이씨 닷 케이알 +.or.kr 닷 오알 닷 케이알 +.go.kr 닷 지오 닷 케이알 +.re.kr 닷 알이 닷 케이알 +.cn 닷 씨엔 +.fr 닷 에프알 +.de 닷 디이 +.it 닷 아이티 +.uk 닷 유케이 +.br 닷 비알 +.in 닷 아이엔 +.ru 닷 알유 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv b/nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv new file mode 100644 index 000000000..c80d08a69 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/electronic/extensions.tsv @@ -0,0 +1,6 @@ +.jpg 닷 제이피지 +.png 닷 피엔지 +.pdf 닷 피디에프 +.JPG 닷 제이피지 +.PNG 닷 피엔지 +.PDF 닷 피디에프 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv b/nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv new file mode 100644 index 000000000..f551dabf4 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/electronic/symbol.tsv @@ -0,0 +1,12 @@ +@ 골뱅이 +. 점 +/ 슬래시 +- 대시 +_ 언더바 +: 콜론 +? 물음표 += 이퀄 +& 앰퍼샌드 +% 퍼센트 ++ 플러스 +# 샵 diff --git a/nemo_text_processing/text_normalization/ko/data/measure/__init__.py b/nemo_text_processing/text_normalization/ko/data/measure/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/measure/unit.tsv b/nemo_text_processing/text_normalization/ko/data/measure/unit.tsv new file mode 100644 index 000000000..ccec41e7f --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/measure/unit.tsv @@ -0,0 +1,18 @@ +kg 킬로그램 +g 그램 +km 킬로미터 +m 미터 +cm 센티미터 +mm 밀리미터 +L 리터 +l 리터 +mL 밀리리터 +ml 밀리리터 +h 시간 +s 초 +N 뉴턴 +W 와트 +Hz 헤르츠 +° 도 +% 퍼센트 +rpm 분당회전수 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/money/__init__.py b/nemo_text_processing/text_normalization/ko/data/money/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/ko/data/money/currency_major.tsv new file mode 100644 index 000000000..a5ddfeb45 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/money/currency_major.tsv @@ -0,0 +1,22 @@ +₩ 원 +KRW 원 +krw 원 +$ 달러 +US$ 달러 +HK$ 홍콩 달러 +hk$ 홍콩 달러 +€ 유로 +EUR 유로 +¥ 엔 +JPY 엔 +CAD 캐나다 달러 +cad 캐나다 달러 +NZD 뉴질랜드 달러 +nzd 뉴질랜드 달러 +CHF 스위스 프랑 +chf 스위스 프랑 +AED 아랍에미리트 디르함 +aed 아랍에미리트 디르함 +Dh 디르함 +DH 디르함 +Dhs. 디르함 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/number/__init__.py b/nemo_text_processing/text_normalization/ko/data/number/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/number/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/number/counter_suffix.tsv b/nemo_text_processing/text_normalization/ko/data/number/counter_suffix.tsv new file mode 100644 index 000000000..6c48f5e7d --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/number/counter_suffix.tsv @@ -0,0 +1,16 @@ +개 +명 +병 +마리 +송이 +포기 +사람 +자루 +채 +켤레 +그루 +벌 +잔 +장 +권 +살 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/number/digit.tsv b/nemo_text_processing/text_normalization/ko/data/number/digit.tsv new file mode 100644 index 000000000..61a7dddcf --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/number/digit.tsv @@ -0,0 +1,9 @@ +1 일 +2 이 +3 삼 +4 사 +5 오 +6 육 +7 칠 +8 팔 +9 구 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/number/native_ones.tsv b/nemo_text_processing/text_normalization/ko/data/number/native_ones.tsv new file mode 100644 index 000000000..f9926b64b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/number/native_ones.tsv @@ -0,0 +1,9 @@ +1 한 +2 두 +3 세 +4 네 +5 다섯 +6 여섯 +7 일곱 +8 여덟 +9 아홉 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/number/tens.tsv b/nemo_text_processing/text_normalization/ko/data/number/tens.tsv new file mode 100644 index 000000000..d8b8e0a2b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/number/tens.tsv @@ -0,0 +1,9 @@ +1 십 +2 이십 +3 삼십 +4 사십 +5 오십 +6 육십 +7 칠십 +8 팔십 +9 구십 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/number/zero.tsv b/nemo_text_processing/text_normalization/ko/data/number/zero.tsv new file mode 100644 index 000000000..7024c0534 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/number/zero.tsv @@ -0,0 +1 @@ +0 영 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/ko/data/ordinal/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/ordinal/digit.tsv b/nemo_text_processing/text_normalization/ko/data/ordinal/digit.tsv new file mode 100644 index 000000000..b3efc4cef --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/ordinal/digit.tsv @@ -0,0 +1,8 @@ +2 두 +3 세 +4 네 +5 다섯 +6 여섯 +7 일곱 +8 여덟 +9 아홉 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/ordinal/exceptions.tsv b/nemo_text_processing/text_normalization/ko/data/ordinal/exceptions.tsv new file mode 100644 index 000000000..ad796a0e8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/ordinal/exceptions.tsv @@ -0,0 +1,5 @@ +1 첫 +11 열한 +20 스무 +21 스물한 +31 서른한 diff --git a/nemo_text_processing/text_normalization/ko/data/ordinal/tens.tsv b/nemo_text_processing/text_normalization/ko/data/ordinal/tens.tsv new file mode 100644 index 000000000..994a21a1d --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/ordinal/tens.tsv @@ -0,0 +1,2 @@ +10 열 +30 서른 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/ordinal/tens_prefix.tsv b/nemo_text_processing/text_normalization/ko/data/ordinal/tens_prefix.tsv new file mode 100644 index 000000000..3111a82dc --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/ordinal/tens_prefix.tsv @@ -0,0 +1,3 @@ +1 열 +2 스물 +3 서른 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/time/__init__.py b/nemo_text_processing/text_normalization/ko/data/time/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/data/time/division.tsv b/nemo_text_processing/text_normalization/ko/data/time/division.tsv new file mode 100644 index 000000000..9250d0a8f --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/time/division.tsv @@ -0,0 +1,7 @@ +오전 +오후 +새벽 +아침 +낮 +저녁 +밤 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/time/hour.tsv b/nemo_text_processing/text_normalization/ko/data/time/hour.tsv new file mode 100644 index 000000000..abfccd310 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/time/hour.tsv @@ -0,0 +1,12 @@ +1 한 +2 두 +3 세 +4 네 +5 다섯 +6 여섯 +7 일곱 +8 여덟 +9 아홉 +10 열 +11 열한 +12 열두 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/data/whitelist.tsv b/nemo_text_processing/text_normalization/ko/data/whitelist.tsv new file mode 100644 index 000000000..d0bdf4caf --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/data/whitelist.tsv @@ -0,0 +1,39 @@ +Dr. 박사 +dr. 박사 +Mr. 씨 +mr. 씨 +Ms. 양 +ms. 양 +Mrs. 여사 +mrs. 여사 +mt. 산 +Mt. 산 +Prof. 교수 +prof. 교수 +sr. 시니어 +Sr. 시니어 +jr. 주니어 +Jr. 주니어 +rd. 로 +Rd. 로 +Ave. 대로 +ave. 대로 +no. 번호 +No. 번호 +( 왼쪽 괄호 +) 오른쪽 괄호 ++ 더하기 +- 마이너스 += 은 +Σ 시그마 +η 에타 +κ 카파 +ω 오메가 +σ 시그마 +α 알파 +ν 뉴 +δ 델타 +ι 이오타 +vs. 대 +Ph.D. 박사학위 +etc. 등 diff --git a/nemo_text_processing/text_normalization/ko/graph_utils.py b/nemo_text_processing/text_normalization/ko/graph_utils.py new file mode 100644 index 000000000..9db51238f --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/graph_utils.py @@ -0,0 +1,173 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import load_labels +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_ALPHA = pynini.union(*[chr(i) for i in range(ord('가'), ord('힣') + 1)]).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + + +# Common string literals; expand as you see fit. +username_string = "username" +double_quotes = '"' +domain_string = "domain" +protocol_string = "protocol" +slash = "/" +double_slash = "//" +triple_slash = "///" +file = "file" +period = "." +at = "@" +colon = ":" +https = "https" +http = "http" +www = "www" + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, "\u00a0"), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = "lower_cased"): + labels = load_labels(input_file) + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/ko/taggers/__init__.py b/nemo_text_processing/text_normalization/ko/taggers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/taggers/cardinal.py b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py new file mode 100644 index 000000000..49c36ec82 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py @@ -0,0 +1,374 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class CardinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + # Optional small whitespace inside parentheses or after signs + ws = pynini.closure(NEMO_SPACE, 0, 2) + + # Load base .tsv files + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + + digit_except_one = pynini.difference(NEMO_DIGIT, "1") + digit_except_zero_one = pynini.difference(digit_except_one, "0") + + graph_digit_no_zero_one = digit_except_zero_one @ graph_digit + graph_tens = pynini.string_file(get_abs_path("data/number/tens.tsv")) + + # Compose all basic number forms + graph_1_to_99 = (graph_tens + (graph_digit | pynutil.delete('0'))) | graph_digit + + hundreds = NEMO_DIGIT**3 + graph_hundred_component = ( + pynini.cross('1', '백') | (graph_digit_no_zero_one + pynutil.insert('백')) + ) + pynini.union(pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_1_to_99)) + graph_hundred = hundreds @ graph_hundred_component + + thousands = NEMO_DIGIT**4 + graph_thousand_component = pynini.union( + pynini.cross('1', '천'), + graph_digit_no_zero_one + pynutil.insert('천'), + ) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_thousand = thousands @ graph_thousand_component + + ten_thousands = NEMO_DIGIT**5 + graph_ten_thousand_component = pynini.union( + pynini.cross('1', '만'), + graph_digit_no_zero_one + pynutil.insert('만'), + ) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_ten_thousand = ten_thousands @ graph_ten_thousand_component + + hundred_thousands = NEMO_DIGIT**6 + graph_hundred_thousand_component = ((NEMO_DIGIT**2 @ graph_1_to_99) + pynutil.insert("만")) + pynini.union( + pynini.closure(pynutil.delete("0")), + graph_thousand_component, + (pynutil.delete("0") + graph_hundred_component), + (pynini.closure(pynutil.delete("0")) + graph_1_to_99), + ) + graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component + + millions = NEMO_DIGIT**7 + graph_million_component = ((graph_hundred) + pynutil.insert('만')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_million = millions @ graph_million_component + + ten_millions = NEMO_DIGIT**8 + graph_ten_million_component = ((graph_thousand) + pynutil.insert('만')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_ten_million = ten_millions @ graph_ten_million_component + + hundred_millions = NEMO_DIGIT**9 + graph_hundred_million_component = (graph_digit + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_hundred_million = hundred_millions @ graph_hundred_million_component + + thousand_millions = NEMO_DIGIT**10 + graph_thousand_million_component = ((NEMO_DIGIT**2 @ graph_1_to_99) + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_thousand_million = thousand_millions @ graph_thousand_million_component + + billions = NEMO_DIGIT**11 + graph_billions_component = ((graph_hundred) + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_billions = billions @ graph_billions_component + + ten_billions = NEMO_DIGIT**12 + graph_ten_billions_component = ((graph_thousand) + pynutil.insert('억')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + graph_million_component), + (pynutil.delete('00') + graph_hundred_thousand_component), + (pynutil.delete('000') + graph_ten_thousand_component), + (pynutil.delete('0000') + graph_thousand_component), + ((pynutil.delete('00000') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_ten_billions = ten_billions @ graph_ten_billions_component + + hundred_billions = NEMO_DIGIT**13 + graph_hundred_billions_component = (graph_digit + pynutil.insert('조')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + graph_hundred_billions = hundred_billions @ graph_hundred_billions_component + + trillion = NEMO_DIGIT**14 + graph_trillion_component = ( + (NEMO_DIGIT**2 @ graph_1_to_99) + + pynutil.insert('조') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_trillions = trillion @ graph_trillion_component + + ten_trillions = NEMO_DIGIT**15 + graph_ten_trillions_component = ( + (graph_hundred) + + pynutil.insert('조') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_ten_trillions = ten_trillions @ graph_ten_trillions_component + + hundred_trillions = NEMO_DIGIT**16 + graph_hundred_trillions_component = ( + (graph_thousand) + + pynutil.insert('조') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_billions_component, + pynutil.delete('0') + graph_billions_component, + pynutil.delete('00') + graph_thousand_million_component, + pynutil.delete('000') + graph_hundred_million_component, + pynutil.delete('0000') + graph_ten_million_component, + pynutil.delete('00000') + graph_million_component, + pynutil.delete('000000') + graph_hundred_thousand_component, + pynutil.delete('0000000') + graph_ten_thousand_component, + pynutil.delete('00000000') + graph_thousand_component, + pynutil.delete('000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_hundred_trillions = hundred_trillions @ graph_hundred_trillions_component + + thousand_trillions = NEMO_DIGIT**17 + graph_thousand_trillions_component = ( + graph_digit + + pynutil.insert('경') + + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_trillions_component, + pynutil.delete('0') + graph_ten_trillions_component, + pynutil.delete('00') + graph_trillion_component, + pynutil.delete('000') + graph_hundred_billions_component, + pynutil.delete('0000') + graph_ten_billions_component, + pynutil.delete('00000') + graph_billions_component, + pynutil.delete('000000') + graph_thousand_million_component, + pynutil.delete('0000000') + graph_hundred_million_component, + pynutil.delete('00000000') + graph_ten_million_component, + pynutil.delete('000000000') + graph_million_component, + pynutil.delete('0000000000') + graph_hundred_thousand_component, + pynutil.delete('00000000000') + graph_ten_thousand_component, + pynutil.delete('000000000000') + graph_thousand_component, + pynutil.delete('0000000000000') + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + graph_1_to_99), + ) + ) + graph_thousand_trillions = thousand_trillions @ graph_thousand_trillions_component + + # FST + graph_num = pynini.union( + graph_thousand_trillions, + graph_hundred_trillions, + graph_ten_trillions, + graph_trillions, + graph_hundred_billions, + graph_ten_billions, + graph_billions, + graph_thousand_million, + graph_hundred_million, + graph_ten_million, + graph_million, + graph_hundred_thousand, + graph_ten_thousand, + graph_thousand, + graph_hundred, + graph_1_to_99, + graph_zero, + ).optimize() + + # ---------------------------- + # Context-based digit-by-digit reading + # e.g., 번호는 0987654321 -> 번호는 영구팔칠육오사삼이일 + # + # Keep this separate from graph_num so regular cardinal zero + # and place-value behavior remain unchanged. + + serial_space = pynini.closure(delete_space) + + # Exclude 0 from graph_digit and force 0 -> 영. + # This avoids ambiguity if digit.tsv has another mapping for 0. + graph_digit_one_to_nine = (pynini.difference(NEMO_DIGIT, "0") @ graph_digit).optimize() + + serial_digit = pynini.union( + pynini.cross("0", "영"), + graph_digit_one_to_nine, + ).optimize() + + # Optional separators between individual digits. + serial_separator = pynini.union( + pynutil.delete("-"), + pynutil.delete("."), + pynutil.delete(" "), + ).optimize() + + # Require at least three digits. + serial_body = ( + (serial_digit + pynini.closure(serial_separator, 0, 1)) ** 2 + + serial_digit + + pynini.closure(pynini.closure(serial_separator, 0, 1) + serial_digit) + ).optimize() + + serial_signal = pynini.string_map( + [ + ("번호는", "번호는 "), + ("번호가", "번호가 "), + ("번호를", "번호를 "), + ("연락처는", "연락처는 "), + ("연락처가", "연락처가 "), + ("연락처를", "연락처를 "), + ] + ).optimize() + + self.serial = ( + pynutil.insert('integer: "') + serial_signal + serial_space + serial_body + pynutil.insert('"') + ).optimize() + + # ---------------------------- + # Native counting + counters + # e.g., 3개, 2명, 10살 + # + # In Korean, counters require native numeral forms + # for small numbers (한/두/세…, 열/스무/서른…). + counter_suffix = pynini.string_file(get_abs_path("data/number/counter_suffix.tsv")) + counter_suffix_accep = pynini.project(counter_suffix, "input").optimize() + + native_ones = pynini.string_file(get_abs_path("data/number/native_ones.tsv")) # 1~9: 한/두/세/... + ordinal_tens = pynini.string_file(get_abs_path("data/ordinal/tens.tsv")) # 10=열, 20=스무, 30=서른 + ordinal_tens_prefix = pynini.string_file(get_abs_path("data/ordinal/tens_prefix.tsv")) # 열/스물/서른 + + native_11_to_39 = (ordinal_tens_prefix + native_ones).optimize() + native_1_to_39 = pynini.union(native_ones, ordinal_tens, native_11_to_39).optimize() + + # Compose number + counter as one cardinal token + counter_case = ( + pynutil.insert('integer: "') + + native_1_to_39 + + pynutil.insert('" ') + + pynutil.insert('counter: "') + + counter_suffix_accep + + pynutil.insert('"') + ).optimize() + + # Sign and final formatting + # Build the integer token (integer: "...") + integer_token = pynutil.insert('integer: "') + graph_num + pynutil.insert('"') + + # Sign handling: + # - minus sets negative flag + # - plus is ignored (positive number) + minus_prefix = pynutil.insert('negative: "true" ') + pynutil.delete("-") + plus_prefix = pynutil.delete("+") + + # Accounting negative: "( 1,234 )" -> negative + integer:"1234" + paren_negative = ( + pynutil.insert('negative: "true" ') + pynutil.delete("(") + ws + integer_token + ws + pynutil.delete(")") + ) + + # Signed number: optional (+|-) + integer + signed_integer = (minus_prefix | plus_prefix).ques + integer_token + + # Prefer accounting-form first, then signed form + final_graph = self.serial | paren_negative | signed_integer | counter_case + + # Wrap with class tokens and finalize + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + self.graph = graph_num diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py new file mode 100644 index 000000000..8b5d89d91 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -0,0 +1,323 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying dates in Korean, e.g. + 2024/01/30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + 2024/1/30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + 2024-01-30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + 2024.01.30 -> date { year: "이천이십사" month: "일월" day: "삼십" } + + 기원전233년 -> date { era: "기원전" year: "이백삼십삼년" } + 기원후2024년 -> date { era: "기원후" year: "이천이십사년" } + + 21일월요일 -> tokens { date { day: "이십일일" weekday: "월요일" } } + 1970년대 -> date { year: "천구백칠십년대" } + + 1월1일(월)~3일(수) + -> tokens { date { month: "일월" day: "일일" weekday: "월요일" } } + tokens { name: "부터" } + tokens { date { day: "삼일" weekday: "수요일" } } + + 1970~1980년대 + -> tokens { cardinal { integer: "천구백칠십" } } + tokens { name: "부터" } + tokens { date { year: "천구백팔십년대" } } + + 7월5~9일(월~금) + -> tokens { date { month: "칠월" } } + tokens { cardinal { integer: "오" } } + tokens { name: "부터" } + tokens { date { day: "구일" weekday: "월요일" } } + tokens { name: "부터" } + tokens { date { weekday: "금요일" } } + + 2023년3월1일(수)~6월12일(화) + -> tokens { date { year: "이천이십삼년" month: "삼월" day: "일일" weekday: "수요일" } } + tokens { name: "부터" } + tokens { date { month: "유월" day: "십이일" weekday: "화요일" } } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + strip0 = pynini.closure(pynutil.delete("0"), 0, 1) + graph_cardinal = cardinal.graph + cardinal_lz = (strip0 + graph_cardinal).optimize() + + # Load base .tsv files + week = pynini.string_file(get_abs_path("data/date/week.tsv")) + month_exceptions = pynini.string_file(get_abs_path("data/date/exceptions.tsv")) + month_exceptions_inputs = pynini.project(month_exceptions, "input").optimize() + + # Non-exception inputs go through the generic cardinal path + graph_cardinal_non_exceptions = pynini.compose( + pynini.difference(pynini.project(graph_cardinal, "input"), month_exceptions_inputs).optimize(), + graph_cardinal, + ).optimize() + + # Month cardinal: prefer exceptions; + month_cardinal = strip0 + (month_exceptions | graph_cardinal_non_exceptions).optimize() + + era = pynini.union("기원전", "기원후").optimize() + signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") + + date_sep = signs + pynini.closure(delete_space, 0, 1) + insert_space + + # Strict digit ranges for M/D/Y and Y/M/D + _d = pynini.union(*[pynini.accep(str(i)) for i in range(10)]) + _1to9 = pynini.union(*[pynini.accep(str(i)) for i in range(1, 10)]) + + # For standalone years: + # - No era: 1–4 digits with NO leading zeros + YEAR_NO_ERA_1TO4 = pynini.closure(pynutil.delete("0"), 0, 3) + _1to9 + pynini.closure(_d, 0, 3) + # - With era (기원전/기원후): allow leading zeros but strip them + YEAR_ERA_1TO4 = pynini.closure(pynutil.delete("0"), 0, 3) + _1to9 + pynini.closure(_d, 0, 3) + + # MM: 01-09 | 10-12 + MM = (pynini.accep("0") + _1to9) | (pynini.accep("1") + pynini.union("0", "1", "2")) + + # DD: 01-09 | 10-19 | 20-29 | 30-31 + DD = ( + (pynini.accep("0") + _1to9) + | (pynini.accep("1") + _d) + | (pynini.accep("2") + _d) + | (pynini.accep("3") + pynini.union("0", "1")) + ) + + # YYYY: exactly 4 digits and two-digit year for M/D/YY and D/M/YY + YYYY = pynini.union("1", "2") + _d + _d + _d + YY = _d + _d + + # Map digits -> cardinal words using existing graphs (strip leading zero via month_cardinal/cardinal_lz) + mm_to_text = pynini.compose(MM, month_cardinal).optimize() + dd_to_text = pynini.compose(DD, cardinal_lz).optimize() + yy_to_text = pynini.compose(YY, graph_cardinal).optimize() + + # Components with tags/suffixes (strict) + month_component_md = ( + pynutil.insert("month: \"") + mm_to_text + pynutil.insert("월") + pynutil.insert("\"") + ).optimize() + day_component_md = ( + pynutil.insert("day: \"") + dd_to_text + pynutil.insert("일") + pynutil.insert("\"") + ).optimize() + year_component_y2 = ( + pynutil.insert("year: \"") + yy_to_text + pynutil.insert("년") + pynutil.insert("\"") + ).optimize() + + # Generic components + era_component = pynutil.insert("era: \"") + era + pynutil.insert("\"") + + # Brackets for weekday + front_bracket = ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete("(") + + pynini.closure(pynutil.delete(delete_space)) + ) | ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete("（") + + pynini.closure(pynutil.delete(delete_space)) + ) + preceding_bracket = ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete(")") + + pynini.closure(pynutil.delete(delete_space)) + ) | ( + pynini.closure(pynutil.delete(delete_space)) + + pynutil.delete("）") + + pynini.closure(pynutil.delete(delete_space)) + ) + + week_component_bracketed = ( + (front_bracket + pynutil.insert("weekday: \"") + week + preceding_bracket + pynutil.insert("\"")) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynini.cross("〜", "부터") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + | ( + front_bracket + + pynutil.insert("weekday: \"") + + week + + pynutil.delete("・") + + week + + preceding_bracket + + pynutil.insert("\"") + ) + ) + + week_component_plain = pynutil.insert("weekday: \"") + week + pynutil.insert("\"") + week_component = week_component_bracketed | week_component_plain + + # Strict 4-digit year component (1000–2999) + year_component_y4_strict = ( + pynutil.insert("year: \"") + (YYYY @ graph_cardinal) + pynutil.insert("년") + pynutil.insert("\"") + ).optimize() + + # Prefer strict 4-digit; still allow 2-digit with worse weight (for MM/DD/YY etc.) + year_component_md_strict = (year_component_y4_strict | pynutil.add_weight(year_component_y2, 1.0)).optimize() + + # Format: YYYY/MM/DD(weekday) + graph_basic_date = ( + pynini.closure(era_component + insert_space, 0, 1) + + year_component_y4_strict + + date_sep + + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\"")) + + date_sep + + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\"")) + + pynini.closure(insert_space + week_component, 0, 1) + ) + graph_basic_date_with_dot_weekday = ( + pynini.closure(era_component + insert_space, 0, 1) + + year_component_y4_strict + + date_sep + + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\"")) + + date_sep + + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\"")) + + pynutil.delete(".") + + pynini.closure(delete_space, 0, 1) + + insert_space + + week_component + ).optimize() + + # American: MM/DD/YYYY + graph_american_date = ( + month_component_md + + signs + + insert_space + + day_component_md + + signs + + insert_space + + year_component_md_strict + + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1) + ).optimize() + + # European: DD/MM/YYYY + graph_european_date = ( + day_component_md + + signs + + insert_space + + month_component_md + + signs + + insert_space + + year_component_md_strict + + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1) + ).optimize() + + # Single elements (year/month/day) + individual_year_component = ( + # with era: (기원전|기원후) + 1~4 digits (leading zeros allowed → stripped) + ( + era_component + + insert_space + + pynutil.insert("year: \"") + + (YEAR_ERA_1TO4 @ graph_cardinal) + + pynini.accep("년") + + pynutil.insert("\"") + ) + | + # no era: 1~4 digits, no leading zero + ( + pynutil.insert("year: \"") + + (YEAR_NO_ERA_1TO4 @ graph_cardinal) + + pynini.accep("년") + + pynutil.insert("\"") + ) + ).optimize() + + individual_month_component = ( + pynutil.insert("month: \"") + month_cardinal + pynini.accep("월") + pynutil.insert("\"") + ) + + month_josa = pynini.union("에", "은", "는", "에는") + + individual_month_component_with_josa = ( + pynutil.insert('month: "') + + month_cardinal + + pynini.accep("월") + + pynini.closure(month_josa, 0, 1) + + pynutil.insert('"') + ).optimize() + + individual_day_component = pynutil.insert("day: \"") + cardinal_lz + pynini.accep("일") + pynutil.insert("\"") + + week_full_word_acceptor = pynini.project(week, "output") + week_component_full_word = pynutil.insert("weekday: \"") + week_full_word_acceptor + pynutil.insert("\"") + + day_and_weekday_component = ( + individual_day_component + pynini.closure(insert_space, 0, 1) + week_component_full_word + ) + + month_and_weekday_component = ( + individual_month_component + pynini.closure(insert_space, 0, 1) + week_component_full_word + ) + + graph_individual_component = ( + day_and_weekday_component + | month_and_weekday_component + | individual_year_component + | individual_month_component_with_josa + | individual_month_component + | individual_day_component + | week_component + ) + pynini.closure(insert_space + week_component, 0, 1) + + graph_individual_component_combined = ( + (individual_year_component + insert_space + individual_month_component) + | (individual_month_component + insert_space + individual_day_component) + | ( + individual_year_component + + insert_space + + individual_month_component + + insert_space + + individual_day_component + ) + ) + pynini.closure(insert_space + week_component, 0, 1) + + nendai = pynini.accep("년대") + era_nendai = ( + pynini.closure(era_component + insert_space, 0, 1) + + pynutil.insert("year: \"") + + graph_cardinal + + nendai + + pynutil.insert("\"") + ).optimize() + + graph_all_date = ( + graph_basic_date_with_dot_weekday + | graph_basic_date + | graph_american_date + | graph_european_date + | graph_individual_component + | graph_individual_component_combined + | era_nendai + ).optimize() + + final_graph = self.add_tokens(graph_all_date) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/decimal.py b/nemo_text_processing/text_normalization/ko/taggers/decimal.py new file mode 100644 index 000000000..f18fe531f --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/decimal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying decimal numbers in Korean, e.g. + 1.23 -> decimal { integer_part: "일" fractional_part: "이삼" } + -0.5 -> decimal { negative: "마이너스" integer_part: "영" fractional_part: "오" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + # Use the base cardinal graph for the integer part + base_integer_graph = cardinal.graph + # Only special-case 10000 -> 만 for decimal integer part (if needed) + specials_input = pynini.cross("10000", "만") + + # Try the special mapping first, then fall back to normal cardinal + cardinal_before_decimal = (specials_input | base_integer_graph).optimize() + + cardinal_after_decimal = pynini.string_file(get_abs_path("data/number/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + DOUBLE_QUOTE = '"' + + graph_integer = ( + pynutil.insert(f'integer_part: {DOUBLE_QUOTE}') + cardinal_before_decimal + pynutil.insert(DOUBLE_QUOTE) + ) + graph_fractional = ( + pynutil.insert(f'fractional_part: {DOUBLE_QUOTE}') + + pynini.closure(cardinal_after_decimal | zero, 1) + + pynutil.insert(DOUBLE_QUOTE) + ) + + # Decimal without a sign (e.g., 2.5) + graph_decimal_no_sign = graph_integer + pynutil.delete('.') + pynutil.insert(NEMO_SPACE) + graph_fractional + + # Negative sign handling (e.g., -2.5 or 마이너스2.5) + graph_with_negative = ( + pynutil.insert(f'negative: {DOUBLE_QUOTE}') + + (pynini.cross("-", "마이너스") | pynini.accep("마이너스")) + + pynutil.insert(DOUBLE_QUOTE) + ) + + graph_decimal = graph_decimal_no_sign | ( + graph_with_negative + pynutil.insert(NEMO_SPACE) + graph_decimal_no_sign + ) + + # For internal use without tokens + self.just_decimal = graph_decimal_no_sign.optimize() + + # Final graph with tokens + final_graph = self.add_tokens(graph_decimal) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/electronic.py b/nemo_text_processing/text_normalization/ko/taggers/electronic.py new file mode 100644 index 000000000..f928448af --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/electronic.py @@ -0,0 +1,168 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer (FST) for classifying **electronic expressions** such as + email addresses, URLs, and domain names in Korean. + + Example conversions: + - abc@nvidia.co.kr → electronic { username: "abc" domain: "nvidia.co.kr" } + - www.nvidia.com → electronic { domain: "www.nvidia.com" } + - https://nvidia.com → electronic { protocol: "HTTPS colon slash slash" domain: "nvidia.com" } + - 1234-5678-9012-3456 → electronic { protocol: "credit card" domain: "1234567890123456" } + + Args: + cardinal: FST for digit/number verbalization (used for numeric parts if non-deterministic). + deterministic: If True, provides a single transduction path; otherwise allows multiple. + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + # ---------- Basic character ranges and symbols ---------- + LOWER = pynini.union(*[pynini.accep(c) for c in "abcdefghijklmnopqrstuvwxyz"]) + UPPER = pynini.union(*[pynini.accep(c) for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"]) + ASCII_LETTER = (LOWER | UPPER).optimize() + ASCII_ALNUM = (ASCII_LETTER | NEMO_DIGIT).optimize() + + HYPHEN = pynini.accep("-") + DOT = pynini.accep(".") + SLASH = pynini.accep("/") + AT = pynini.accep("@") + + # Handle numeric reading mode (only for non-deterministic mode) + numbers = ( + NEMO_DIGIT + if deterministic + else (pynutil.insert(NEMO_SPACE) + cardinal.long_numbers + pynutil.insert(NEMO_SPACE)) + ) + + # ---------- Load resources ---------- + cc_cues = pynini.string_file(get_abs_path("data/electronic/cc_cues.tsv")) + accepted_symbols = pynini.project(pynini.string_file(get_abs_path("data/electronic/symbol.tsv")), "input") + graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize() + + # ---------- Username ---------- + # Exclude '@' from username + username_symbols = pynini.difference(accepted_symbols, AT) + # Start with alphanumeric and allow symbols/numbers repeatedly + username_core = ASCII_ALNUM + pynini.closure(ASCII_ALNUM | numbers | username_symbols) + username = pynutil.insert('username: "') + username_core + pynutil.insert('"') + pynini.cross("@", NEMO_SPACE) + + # ---------- Domain ---------- + # Simplified RFC: label = [A-Za-z0-9-]+ , TLD = '.' [A-Za-z0-9]{2,} + label = pynini.closure(ASCII_ALNUM | HYPHEN, 1) + tld = DOT + pynini.closure(ASCII_ALNUM, 2) + # Domain can be (label + TLD) or TLD only (e.g., ".com") + domain_core = (label + pynini.closure(tld, 1)) | tld + + # Optional path after domain (e.g., /path) + path_segment = pynini.closure(NEMO_NOT_SPACE, 1) # at least one non-space character + path = SLASH + path_segment # / + optional_path = pynini.closure(path, 0, 1) # optional path + + domain_with_opt_path = domain_core + optional_path + + domain_graph_with_class_tags = ( + pynutil.insert('domain: "') + domain_with_opt_path.optimize() + pynutil.insert('"') + ) + + # ---------- protocol ---------- + protocol_symbols = pynini.closure((graph_symbols | pynini.cross(":", "colon")) + pynutil.insert(NEMO_SPACE)) + protocol_start = (pynini.cross("https", "HTTPS ") | pynini.cross("http", "HTTP ")) + ( + pynini.accep("://") @ protocol_symbols + ) + protocol_file_start = pynini.accep("file") + insert_space + (pynini.accep(":///") @ protocol_symbols) + protocol_end = pynutil.add_weight(pynini.cross("www", "WWW ") + pynini.accep(".") @ protocol_symbols, -1000) + protocol = protocol_file_start | protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"') + + # ---------- Combine all graphs ---------- + graph = pynini.Fst() # empty + + # (1) Email pattern + email_guard = NEMO_SIGMA + AT + NEMO_SIGMA + DOT + NEMO_SIGMA + graph |= pynini.compose(email_guard, username + domain_graph_with_class_tags) + + # (2) Domain only (without protocol)) + # Domain core graph + graph_domain = (pynutil.insert('domain: "') + domain_core + pynutil.insert('"')).optimize() + graph |= graph_domain + + known_extensions = pynini.project( + pynini.string_file(get_abs_path("data/electronic/extensions.tsv")), + "input", + ) + + filename_stem = pynini.closure( + pynini.difference(NEMO_NOT_SPACE, pynini.union(SLASH, DOT)), + 1, + ) + + file_with_extension = filename_stem + known_extensions + + graph |= (pynutil.insert('domain: "') + file_with_extension + pynutil.insert('"')).optimize() + + # (3) URL with protocol + graph |= protocol + insert_space + domain_graph_with_class_tags + + # (4) Credit card pattern: cue + 4–16 digits + if deterministic: + cc_digits = pynini.closure(NEMO_DIGIT, 4, 16) + cc_phrases = ( + pynutil.insert('protocol: "') + + cc_cues + + pynutil.insert('" domain: "') + + delete_space + + cc_digits + + pynutil.insert('"') + ) + graph |= cc_phrases + + four = pynini.closure(NEMO_DIGIT, 4, 4) + sep_token = pynini.union(HYPHEN, NEMO_SPACE) + cc16_grouped = (four + pynini.cross(sep_token, " ")) ** 3 + four + cc16_grouped = cc16_grouped + delete_space + + cc16_no_cue = ( + pynutil.insert('protocol: "신용카드 " ') + + pynutil.insert('domain: "') + + cc16_grouped + + pynutil.insert('"') + ) + + # Give it higher priority over Date FST + cc16_no_cue = pynutil.add_weight(cc16_no_cue.optimize(), -1.0) + + graph |= cc16_no_cue + + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/fraction.py b/nemo_text_processing/text_normalization/ko/taggers/fraction.py new file mode 100644 index 000000000..64ea0c56e --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/fraction.py @@ -0,0 +1,113 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_SPACE, GraphFst, delete_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying Korean fractions, e.g. + 3/5 → tokens { fraction { numerator: "삼" denominator: "오" } } + 2과7/9 → tokens { fraction { integer_part: "이" numerator: "칠" denominator: "구" } } + 마이너스3/5 → tokens { fraction { negative: "마이너스" numerator: "삼" denominator: "오" } } + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinal = cardinal.graph + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + DOUBLE_QUOTE = '"' + slash = pynutil.delete('/') + root = pynini.accep('√') + + # Decimal number (e.g., 1.23 → 일점이삼) + decimal_number = cardinal + pynini.cross(".", "점") + pynini.closure(graph_digit | graph_zero) + + # Accept cardinal / root + cardinal / decimal / root + decimal + numeral = cardinal | (root + cardinal) | decimal_number | (root + decimal_number) + + # Integer part (e.g., 2과, 1와) + integer_component = ( + pynutil.insert(f'integer_part: {DOUBLE_QUOTE}') + + numeral + + (pynini.accep("과") | pynini.accep("와")) + + pynutil.insert(DOUBLE_QUOTE) + ) + + integer_component_with_space = integer_component + delete_space + pynutil.insert(NEMO_SPACE) + + # Denominator and numerator + denominator_component = pynutil.insert(f'denominator: {DOUBLE_QUOTE}') + numeral + pynutil.insert(DOUBLE_QUOTE) + + numerator_component = pynutil.insert(f'numerator: {DOUBLE_QUOTE}') + numeral + pynutil.insert(DOUBLE_QUOTE) + + # Format 1: 3/4 style + graph_fraction_slash = ( + pynini.closure(integer_component_with_space, 0, 1) + + numerator_component + + slash + + pynutil.insert(NEMO_SPACE) + + denominator_component + + pynutil.insert(NEMO_SPACE) + + pynutil.insert('morphosyntactic_features: "분의"') + ) + + # Format 2: Korean native "4분의3" style + graph_fraction_word = ( + pynini.closure(integer_component_with_space, 0, 1) + + denominator_component + + pynutil.delete("분의") + + pynutil.insert(NEMO_SPACE) + + pynutil.insert('morphosyntactic_features: "분의"') + + pynutil.insert(NEMO_SPACE) + + numerator_component + ) + + # Optional particles following the fraction + particle_subject = pynutil.insert('morphosyntactic_features: "분의_subject"') + ( + pynutil.delete("이") | pynutil.delete("가") + ) + particle_topic = pynutil.insert('morphosyntactic_features: "분의_topic"') + ( + pynutil.delete("은") | pynutil.delete("는") + ) + particle_object = pynutil.insert('morphosyntactic_features: "분의_object"') + ( + pynutil.delete("을") | pynutil.delete("를") + ) + + optional_particle = pynini.closure( + pynutil.insert(NEMO_SPACE) + (particle_subject | particle_topic | particle_object), + 0, + 1, + ) + + # Optional minus sign + optional_sign = ( + pynutil.insert(f'negative: {DOUBLE_QUOTE}') + + (pynini.accep("마이너스") | pynini.cross("-", "마이너스")) + + pynutil.insert(DOUBLE_QUOTE) + + pynutil.insert(NEMO_SPACE) + ) + + # Combine full graph + graph = pynini.closure(optional_sign, 0, 1) + (graph_fraction_slash | graph_fraction_word) + optional_particle + self.graph = graph.optimize() + final_graph = self.add_tokens(graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/measure.py b/nemo_text_processing/text_normalization/ko/taggers/measure.py new file mode 100644 index 000000000..0891e5783 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/measure.py @@ -0,0 +1,106 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying Korean measure expressions. + - 1kg → measure { cardinal { integer: "일" } units: "킬로그램" } + - 12.5km → measure { decimal { integer_part: "십이" fractional_part: "오" } units: "킬로미터" } + - 2/3m → measure { fraction { numerator: "이" denominator: "삼" } units: "미터" } + - 60km/h → measure { cardinal { integer: "육십" } units: "킬로미터 퍼 시간" } + + This FST attaches measurement units (e.g., "킬로미터", "그램") to numeric expressions + classified by the `cardinal`, `decimal`, or `fraction` subgraphs. + + Args: + cardinal: FST handling integer (cardinal) numbers. + decimal: FST handling decimal numbers (optional). + fraction: FST handling fractional numbers (optional). + deterministic: If True, provides a single transduction path; otherwise allows multiple. + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst = None, + fraction: GraphFst = None, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + # Numeric subgraphs + graph_cardinal = cardinal.graph + + # Unit lexicon + graph_unit = pynini.string_file(get_abs_path("data/measure/unit.tsv")) + + # Per-expression handling (e.g., km/h, m/s) + opt_space = pynini.closure(delete_space, 0, 1) + per = pynini.cross("/", "퍼") + opt_space + insert_space + graph_unit + optional_per = pynini.closure(opt_space + insert_space + per, 0, 1) + + # Final unit FST produces either "" or " 퍼 " + unit = pynutil.insert('units: "') + (graph_unit + optional_per | per) + pynutil.insert('"') + + minus_as_field = pynutil.insert('negative: "마이너스" ') + consume_minus = pynini.cross("-", "") | pynini.cross("마이너스", "") + + # Optional minus field + removal of actual sign symbol or word + optional_minus = pynini.closure(minus_as_field + consume_minus + opt_space, 0, 1) + + # Combine numeric and unit components + pieces = [] + + # 1) Cardinal form: e.g., "12kg" + sub_cardinal = ( + pynutil.insert("cardinal { ") + + pynutil.insert('integer: "') + + graph_cardinal + + delete_space + + pynutil.insert('" } ') + + unit + ) + pieces.append(sub_cardinal) + + # 2) Decimal form: e.g., "12.5km" + if decimal is not None: + sub_decimal = ( + pynutil.insert("decimal { ") + + optional_minus + + decimal.just_decimal + + delete_space + + pynutil.insert(" } ") + + unit + ) + pieces.append(sub_decimal) + + # 3) Fraction form: e.g., "2/3m" or "삼분의 이 미터" + if fraction is not None: + sub_fraction = pynutil.insert("fraction { ") + fraction.graph + delete_space + pynutil.insert(" } ") + unit + pieces.append(sub_fraction) + + # Union all supported numeric forms (cardinal | decimal | fraction) + graph = pieces[0] + for p in pieces[1:]: + graph |= p + + # Final wrapping into tokens { measure { ... } } + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/money.py b/nemo_text_processing/text_normalization/ko/taggers/money.py new file mode 100644 index 000000000..a6a200184 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/money.py @@ -0,0 +1,99 @@ +# Copyright (c) 2025 NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying Korean money. + + Example inputs and outputs: + ₩350 -> money { currency_maj: "원" integer_part: "삼백오십" } + 350원 -> money { integer_part: "삼백오십" currency_maj: "원" } + KRW 12,050 -> money { currency_maj: "원" integer_part: "일만이천오십" } + 12만 500원 -> money { integer_part: "십이만오백" currency_maj: "원" } + ₩10.25 -> money { currency_maj: "원" integer_part: "십" minor_part: "이십오" } # optional 2-digit minor + 0원 -> money { integer_part: "영" currency_maj: "원" } + + Args: + cardinal: CardinalFst + deterministic: If True, provide a single transduction; + if False, allow multiple transductions. + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.graph + sp = pynini.closure(delete_space) # absorb any amount of spaces in input + + # --- Numbers (integer / optional minor) --- + # Integer part: "0" or a non-zero leading digit; allow commas (e.g., 18,925,000) + integer_part_fst = pynini.union("0", (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT | pynutil.delete(","))) + + # Plain integer → integer_part: "" + graph_integer_plain = ( + pynutil.insert('integer_part: "') + (integer_part_fst @ graph_cardinal) + pynutil.insert('" ') + ) + + # Optional 2-digit decimal (kept as minor_part if ever used downstream) + decimal_part_fst = NEMO_DIGIT**2 + graph_minor = pynutil.insert('minor_part: "') + (decimal_part_fst @ graph_cardinal) + pynutil.insert('" ') + + # Integer with scale suffix (만/억/조) → wrap the whole thing in one integer_part + scale_unit = pynini.union("만", "억", "조") + value_with_scale = (integer_part_fst @ graph_cardinal) + scale_unit + graph_integer_with_suffix = ( + pynutil.insert('integer_part: "') + value_with_scale + pynutil.insert('" ') + ).optimize() + + # Integer (+ optional ".<2-digit>" minor) + number_component_plain = graph_integer_plain + pynini.closure(pynutil.delete(".") + graph_minor, 0, 1) + number_component = (graph_integer_with_suffix | number_component_plain).optimize() + + # --- Currency (prefix or suffix) --- + # currency_major.tsv example: + # ₩ 원 + # KRW 원 + # 원 원 + maj_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + + # Prefix currency (e.g., ₩, KRW): emit currency_maj then number + currency_major_prepended = pynini.union( + *[pynutil.delete(surface) + pynutil.insert(f'currency_maj: "{unit}" ') for surface, unit in maj_labels] + ).optimize() + + # Suffix currency (e.g., ...원, ...달러): convert unit literal to currency_maj + currency_major_appended = pynini.union( + *[pynutil.delete(unit) + pynutil.insert(f'currency_maj: "{unit}" ') for _, unit in maj_labels] + ).optimize() + + # --- Compose (NO period handling) --- + # NOTE: We deliberately do NOT consume '/월', '/년', '/주', '/일', '/시간' here. + # If present in the raw text, they remain outside the money token and can be handled upstream/elsewhere. + + # [currency] [number] + graph_prepend = (currency_major_prepended + sp + number_component).optimize() + + # [number] [currency] + graph_append = (number_component + currency_major_appended).optimize() + + graph = (graph_prepend | graph_append).optimize() + + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/ordinal.py b/nemo_text_processing/text_normalization/ko/taggers/ordinal.py new file mode 100644 index 000000000..59fa30ada --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/ordinal.py @@ -0,0 +1,85 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying Korean ordinal expressions, e.g. + 1번째 -> ordinal { integer: "첫번째" } + + Args: + cardinal: CardinalFst + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + # Load base .tsv files + graph_digit = pynini.string_file(get_abs_path("data/ordinal/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + graph_exceptions = pynini.string_file(get_abs_path("data/ordinal/exceptions.tsv")) + graph_tens = pynini.string_file(get_abs_path("data/ordinal/tens.tsv")) + graph_tens_prefix = pynini.string_file(get_abs_path("data/ordinal/tens_prefix.tsv")) + + graph_11_to_39 = (graph_tens_prefix + graph_digit).optimize() + + # Combine all ordinal forms from 1 to 39 + graph_ordinal_1to39 = ( + graph_exceptions | graph_digit | graph_zero | graph_tens | graph_11_to_39 + ).optimize() + pynini.accep("번째") + + # Accept tens digit 4–9 + tens_digit_4_to_9_accep = pynini.union(*[pynini.accep(str(i)) for i in range(4, 10)]) + # Accept any single digit + any_single_digit_accep = pynini.union(*[pynini.accep(str(i)) for i in range(0, 10)]) + # Combine two digits + from_40_to_99_inputs = tens_digit_4_to_9_accep + any_single_digit_accep + + # Match numbers with 3 or more digits + input_100_plus = pynini.closure(any_single_digit_accep, 3) + + # Combine both ranges (40–99 and 100+): total range = 40 and above + filter_inputs_from_40 = (from_40_to_99_inputs | input_100_plus).optimize() + + # Only allow cardinal numbers that are 40 or more + graph_cardinal_from40_filtered = pynini.compose(filter_inputs_from_40, cardinal.graph) + + # Add "번째" to the filtered cardinal graph. + graph_ordinal_from40 = graph_cardinal_from40_filtered + pynini.accep("번째") + + graph_ordinal = (graph_ordinal_1to39 | graph_ordinal_from40).optimize() # Handles 1-39 # Handles 40+ + + # Single-character particles (가, 이, 은, 는, 로, 도 ...) + josa_single = pynini.union("가", "이", "은", "는", "를", "을", "로", "도", "다") + + # Multi-character particles (부터, 까지) + josa_multi = pynini.union("부터", "까지") + + # Allow patterns like: + # 번째 + (optional single-josa) + (optional multi-josa) + josa = (josa_single.ques + josa_multi.ques).optimize() + + # Final ordinal graph with optional particles + graph_ordinal_with_josa = (graph_ordinal + josa).optimize() + + # Build the “integer: …” token structure + final_graph = pynutil.insert('integer: "') + graph_ordinal_with_josa + pynutil.insert('"') + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/punctuation.py b/nemo_text_processing/text_normalization/ko/taggers/punctuation.py new file mode 100644 index 000000000..a10250a99 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/punctuation.py @@ -0,0 +1,37 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + + range_component = pynini.cross("〜", "부터") | pynini.accep("부터") + + graph = pynutil.insert('name: "') + range_component + pynutil.insert('"') + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/nemo_text_processing/text_normalization/ko/taggers/telephone.py new file mode 100644 index 000000000..e20cfc208 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -0,0 +1,115 @@ +# Copyright (c) 2025 NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying Korean telephone numbers. + + Example inputs → tokens: + +82 010-3713-7050 -> telephone { country_code: "국가번호 팔이," number_part: "영일영 삼칠일삼 칠영오영" } + +1 (415) 555-0123 -> telephone { country_code: "국가번호 일," number_part: "사일오 오오오 영일이삼" } + (031)371-3700 -> telephone { number_part: "영삼일 삼칠일 삼칠영영" } + 010-3713-7050 -> telephone { number_part: "영일영 삼칠일삼 칠영오영" } + 010.777.8888 -> telephone { number_part: "영일영 칠칠칠 팔팔팔팔" } + + Args: + deterministic (bool, optional): If True, provide a single transduction; + if False, allow multiple transductions. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + # Separator between number blocks. + delete_sep = pynutil.delete(pynini.union("-", ".", " ")).optimize() + + # Optional space inserted between blocks + insert_block_space = insert_space + + # 1) safe digit mapping: force 0 -> "영" (do not rely on zero.tsv invert) + digit = pynini.string_file(get_abs_path("data/number/digit.tsv")).optimize() + zero_map = pynini.cross("0", "영") + digit_ko = (digit | zero_map).optimize() + + two_digits = digit_ko**2 + three_digits = digit_ko**3 + four_digits = digit_ko**4 + + # country code: "+1", "+82", "+1-" + cc_digits = pynini.closure(digit_ko, 1, 3) + + country_code = ( + pynutil.delete("+") + + pynutil.insert('country_code: "') + + cc_digits + + pynutil.insert('"') + + pynini.closure(pynutil.delete("-") | pynutil.delete(" "), 0, 1) + + delete_space + ) + + # First block may contain 2 or 3 digits. + # Examples: 02, 031, 043, 010 + first_block = pynini.union( + two_digits, + three_digits, + ).optimize() + + # Middle block may contain 3 or 4 digits. + # Examples: 123, 1234 + middle_block = pynini.union( + three_digits, + four_digits, + ).optimize() + + # Plain telephone form: + # 02-1234-5678 + plain_first_part = (first_block + delete_sep + insert_block_space).optimize() + + # Parenthesized telephone form: + # (010)1234-5678 + parenthesized_first_part = ( + pynutil.delete("(") + + first_block + + pynutil.delete(")") + + pynini.closure( + pynutil.delete(pynini.union(" ", "-", ".")), + 0, + 1, + ) + + insert_block_space + ).optimize() + + first_part = pynini.union( + plain_first_part, + parenthesized_first_part, + ).optimize() + + # Standard telephone layout: + # 2 or 3 digits + # followed by 3 or 4 digits + # followed by 4 digits + number_part_core = (first_part + middle_block + delete_sep + insert_block_space + four_digits).optimize() + + number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') + + # final graph: with or without country code + graph = pynini.union(country_code + insert_space + number_part, number_part).optimize() + + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/time.py b/nemo_text_processing/text_normalization/ko/taggers/time.py new file mode 100644 index 000000000..b8a499823 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/time.py @@ -0,0 +1,192 @@ +# Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time, e.g. + 오전 10시 30분 -> time { suffix: "오전" hours: "열시" minutes: "삼십분" } + 오후 3시 반 -> time { suffix: "오후" hours: "세시" minutes: "삼십분" } + 자정 -> time { hours: "영시" } + 정오 -> time { hours: "열두시" } + + Args: + cardinal: CardinalFst (Korean cardinal graph) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + # Base number-to-words graph from the Cardinal Fst + graph_cardinal = cardinal.graph + strip0 = pynini.closure(pynutil.delete("0"), 0, 1) + + SP = pynini.closure(delete_space) + SEP = SP + insert_space + hour_clock = pynini.string_file(get_abs_path("data/time/hour.tsv")) + division = pynini.string_file(get_abs_path("data/time/division.tsv")) + + division_component = pynutil.insert("suffix: \"") + division + pynutil.insert("\"") + + # --- Special single-word times --- + noon = pynini.accep("정오") + midnight = pynini.accep("자정") + noon_component = pynutil.insert("hours: \"") + pynini.cross(noon, "열두시") + pynutil.insert("\"") + midnight_component = pynutil.insert("hours: \"") + pynini.cross(midnight, "영시") + pynutil.insert("\"") + + # --- Korean Hangul components (H시 [M분|반] [S초]) --- + # "0" or "00" -> 0 + h_zero = strip0 + pynini.accep("0") + # "13".."24" + h_13_24 = pynini.union(*[str(i) for i in range(13, 25)]) + + # "0시" -> "영시" + hour_component_ko_zero = ( + pynutil.insert("hours: \"") + + pynini.cross(h_zero, "영") + + pynutil.delete("시") + + pynutil.insert("시") + + pynutil.insert("\"") + ) + + # "13시..24시" -> Sino-Korean words (십삼/…/이십사) + 시 + hour_component_ko_13_24 = ( + pynutil.insert("hours: \"") + + (h_13_24 @ graph_cardinal) + + pynutil.delete("시") + + pynutil.insert("시") + + pynutil.insert("\"") + ) + + # "1시..12시" -> Native Korean words (한/두/세/네/…/열두) + 시 + hour_component_ko_1_12 = ( + pynutil.insert("hours: \"") + + (strip0 + hour_clock) + + pynutil.delete("시") + + pynutil.insert("시") + + pynutil.insert("\"") + ) + + # Priority: 13-24 > 0 > 1-12 to prevent partial matching errors + hour_component_ko = (hour_component_ko_13_24 | hour_component_ko_zero | hour_component_ko_1_12).optimize() + + # Minutes: number+"분" or "반" (approx. 30분). Allows optional '쯤|경' after minutes/반. + about_word = pynini.union("쯤", "경") + minute_number = ( + pynutil.insert("minutes: \"") + + (strip0 + graph_cardinal) + + pynutil.delete("분") + + pynutil.insert("분") + + pynutil.insert("\"") + ) + minute_half = ( + pynutil.insert("minutes: \"") + + pynutil.delete("반") + + pynutil.insert("반") + + pynini.closure(about_word, 0, 1) + + pynutil.insert("\"") + ) + minute_component_ko = (minute_half | minute_number).optimize() + + second_component_ko = ( + pynutil.insert("seconds: \"") + + (strip0 + graph_cardinal) + + pynutil.delete("초") + + pynutil.insert("초") + + pynutil.insert("\"") + ) + + # Allow suffix before or after + suffix_prefix_opt = pynini.closure(division_component + SEP, 0, 1) + suffix_postfix_opt = pynini.closure(SEP + division_component, 0, 1) + + # Hangul patterns + graph_hangul = ( + suffix_prefix_opt + + ( + hour_component_ko + | (hour_component_ko + SEP + minute_component_ko) + | (hour_component_ko + SEP + minute_component_ko + SEP + second_component_ko) + | minute_component_ko + | (minute_component_ko + SEP + second_component_ko) + | second_component_ko + ) + + suffix_postfix_opt + ).optimize() + + # Special words with optional suffix + graph_special = (suffix_prefix_opt + (noon_component | midnight_component) + suffix_postfix_opt).optimize() + + # --- Clock patterns: HH:MM[:SS] --- + colon = pynutil.delete(":") + + # Map 1-12 hours using native-Korean words, allowing an optional leading zero. + graph_hour_1_12 = ( + pynutil.insert("hours: \"") + (strip0 + hour_clock) + pynutil.insert("시") + pynutil.insert("\"") + ).optimize() + + # 0, 00, and 13-24 -> Sino-Korean words + hour_sino_val = ( + pynini.cross("00", "0") + | pynini.cross("0", "0") + | pynini.union(*[pynini.cross(str(i), str(i)) for i in range(13, 25)]) + ) + hour_sino_read = hour_sino_val @ graph_cardinal + + graph_hour_others = pynutil.insert("hours: \"") + hour_sino_read + pynutil.insert("시") + pynutil.insert("\"") + + hour_clock_component = (graph_hour_1_12 | graph_hour_others).optimize() + + minute_clock_component = ( + pynutil.insert("minutes: \"") + strip0 + graph_cardinal + pynutil.insert("분") + pynutil.insert("\"") + ) + second_clock_component = ( + pynutil.insert("seconds: \"") + strip0 + graph_cardinal + pynutil.insert("초") + pynutil.insert("\"") + ) + + # HH:MM (drop minutes if "00") + graph_hm_clock = ( + suffix_prefix_opt + + hour_clock_component + + delete_space.ques + + colon + + delete_space.ques + + (pynini.cross("00", "") | pynini.closure(insert_space + minute_clock_component, 0, 1)) + + suffix_postfix_opt + ).optimize() + + # HH:MM:SS (drop minutes/seconds if "00") + graph_hms_clock = ( + suffix_prefix_opt + + hour_clock_component + + delete_space.ques + + colon + + delete_space.ques + + (pynini.cross("00", "") | pynini.closure(insert_space + minute_clock_component, 0, 1)) + + delete_space.ques + + colon + + delete_space.ques + + (pynini.cross("00", "") | pynini.closure(insert_space + second_clock_component, 0, 1)) + + suffix_postfix_opt + ).optimize() + + graph = (graph_special | graph_hangul | graph_hm_clock | graph_hms_clock).optimize() + graph_final = self.add_tokens(graph) + self.fst = graph_final.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..e2a3a5890 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py @@ -0,0 +1,114 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.ko.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ko.taggers.date import DateFst +from nemo_text_processing.text_normalization.ko.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ko.taggers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.ko.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.ko.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.ko.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.ko.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ko.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.ko.taggers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.ko.taggers.time import TimeFst +from nemo_text_processing.text_normalization.ko.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.ko.taggers.word import WordFst +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str = "cased", + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ko_tn_{deterministic}_tokenize.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + cardinal = CardinalFst(deterministic=deterministic) + date = DateFst(cardinal=cardinal, deterministic=deterministic) + time = TimeFst(cardinal=cardinal, deterministic=deterministic) + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + word = WordFst(deterministic=deterministic) + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + punctuation = PunctuationFst(deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + electronic = ElectronicFst(cardinal=cardinal, deterministic=deterministic) + + classify = pynini.union( + pynutil.add_weight(cardinal.fst, 1.1), + pynutil.add_weight(date.fst, 1.1), + pynutil.add_weight(time.fst, 1.1), + pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(ordinal.fst, 1.1), + pynutil.add_weight(decimal.fst, 1.05), + pynutil.add_weight(word.fst, 100), + pynutil.add_weight(money.fst, 1.1), + pynutil.add_weight(measure.fst, 1.1), + pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(whitelist.fst, 1.1), + pynutil.add_weight(telephone.fst, 1.1), + pynutil.add_weight(electronic.fst, 1.11), + ) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + + graph = ( + delete_space + token + pynini.closure((delete_extra_space | pynini.accep("")) + token) + delete_space + ) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/nemo_text_processing/text_normalization/ko/taggers/whitelist.py b/nemo_text_processing/text_normalization/ko/taggers/whitelist.py new file mode 100644 index 000000000..8977d8c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/whitelist.py @@ -0,0 +1,29 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class WhiteListFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + graph = pynutil.insert('name: "') + whitelist + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/word.py b/nemo_text_processing/text_normalization/ko/taggers/word.py new file mode 100644 index 000000000..7aa3db709 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/word.py @@ -0,0 +1,36 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Korean word. + e.g. 이름 -> tokens { name: "이름" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + word_char = pynini.difference(NEMO_NOT_SPACE, NEMO_DIGIT) + + word = pynutil.insert('name: "') + word += pynini.closure(word_char, 1) + word += pynutil.insert('"') + + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/ko/utils.py b/nemo_text_processing/text_normalization/ko/utils.py new file mode 100644 index 000000000..51aaea3e8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/utils.py @@ -0,0 +1,60 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +def augment_labels_with_punct_at_end(labels): + """ + augments labels: if key ends on a punctuation that value does not have, add a new label + where the value maintains the punctuation + + Args: + labels : input labels + Returns: + additional labels + """ + res = [] + for label in labels: + if len(label) > 1: + if label[0][-1] == "." and label[1][-1] != ".": + res.append([label[0], label[1] + "."] + label[2:]) + return res diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/__init__.py b/nemo_text_processing/text_normalization/ko/verbalizers/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/ko/verbalizers/cardinal.py new file mode 100644 index 000000000..d2adfa7d9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/cardinal.py @@ -0,0 +1,53 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> 마이너스 이십삼 + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "마이너스 ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "음수 ") + self.optional_sign |= pynini.cross("negative: \"true\"", "- ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + # quoted: parses a quoted string value like "십", "명" + quoted = delete_space + pynutil.delete('"') + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('"') + + # integer: mandatory field + integer = pynutil.delete("integer:") + quoted + + # counter: optional field (e.g., 명/개/살). + counter = pynutil.delete("counter:") + quoted + counter = pynini.closure(delete_space + counter, 0, 1) + self.numbers = self.optional_sign + integer + counter + + delete_tokens = self.delete_tokens(self.numbers) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/date.py b/nemo_text_processing/text_normalization/ko/verbalizers/date.py new file mode 100644 index 000000000..bfd5e9aa1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/date.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class DateFst(GraphFst): + """ + Korean date verbalizer + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + era_component = pynutil.delete("era: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + year_component = pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + month_component = pynutil.delete("month: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + day_component = pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + week_component = ( + pynutil.delete("weekday: \"") + + pynini.closure(delete_space) + + pynini.closure(NEMO_NOT_QUOTE) + + pynini.closure(delete_space) + + pynutil.delete("\"") + ) + + SPACE = pynini.closure(delete_space, 0, 1) + insert_space + + # This graph now correctly uses the 'delete_space' variable defined above. + graph_basic_date = ( + pynini.closure(era_component + SPACE, 0, 1) + + pynini.closure(year_component + SPACE, 0, 1) + + pynini.closure(month_component + SPACE, 0, 1) + + pynini.closure(day_component, 0, 1) + + pynini.closure(SPACE + week_component, 0, 1) + ) | (month_component + SPACE + week_component) + + final_graph = graph_basic_date + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/decimal.py b/nemo_text_processing/text_normalization/ko/verbalizers/decimal.py new file mode 100644 index 000000000..54375c5a9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/decimal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class DecimalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + # Extract integer part + integer_part = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + # Extract fractional part and prepend "점" + fractional_part = ( + pynutil.delete('fractional_part: "') + + pynutil.insert("점") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + # Verbalize decimal number without sign + decimal_positive = integer_part + pynutil.delete(" ") + fractional_part + + # Handle negative sign + negative_sign = ( + pynutil.delete('negative: "') + pynini.accep("마이너스") + pynutil.delete('"') + pynutil.delete(" ") + ) + + # Combine positive and negative cases + decimal = decimal_positive | (negative_sign + pynutil.insert(" ") + decimal_positive) + + delete_tokens = self.delete_tokens(decimal) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/electronic.py b/nemo_text_processing/text_normalization/ko/verbalizers/electronic.py new file mode 100644 index 000000000..dc3b3f97b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/electronic.py @@ -0,0 +1,141 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_ALPHA, + NEMO_CHAR, + NEMO_DIGIT, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer (FST) for verbalizing **electronic expressions** (email/URL/domain). + + Input tokens: + tokens { electronic { username: "abc" domain: "abc.com" } } + + Example output (policy-dependent): + abc 골뱅이 abc 닷컴 + + Args: + deterministic: If True, produce a single verbalization. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + # 1) Handle digits (0–9) + graph_digit_no_zero = pynini.string_file(get_abs_path("data/number/digit.tsv")).optimize() + + graph_zero = pynini.cross("0", "영") + if not deterministic: + graph_zero |= pynini.cross("0", "공") + graph_digit = (graph_digit_no_zero | graph_zero).optimize() + + digit_inline_rewrite = pynini.cdrewrite( + graph_digit, + "", + "", + NEMO_SIGMA, + ) + + # 3) username part (add spaces between characters) + raw_username = pynini.closure(NEMO_NOT_QUOTE, 1) + + user_name = ( + pynutil.delete("username:") + + delete_space + + pynutil.delete('"') + + (raw_username @ digit_inline_rewrite) + + pynutil.delete('"') + ) + + # 4) domain part (handle common endings like .com → 닷컴) + domain_common_pairs = ( + pynini.string_file(get_abs_path("data/electronic/domain.tsv")) + | pynini.string_file(get_abs_path("data/electronic/extensions.tsv")) + ).optimize() + + # Rewrite known domains (.com → 닷컴) + tld_rewrite = pynini.cdrewrite( + domain_common_pairs, + "", + "", + NEMO_SIGMA, + ) + # Add a space before “닷” if needed + add_space_before_dot = pynini.cdrewrite( + pynini.cross("닷", " 닷"), + (NEMO_ALPHA | NEMO_DIGIT | NEMO_CHAR), + "", + NEMO_SIGMA, + ) + + raw_domain = pynini.closure(NEMO_NOT_QUOTE, 1) + + four = pynini.closure(NEMO_DIGIT, 4, 4) + cc16_grouped = four + pynutil.insert(" ") + four + pynutil.insert(" ") + four + pynutil.insert(" ") + four + cc_domain = (cc16_grouped @ digit_inline_rewrite).optimize() + + domain = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete('"') + + ((raw_domain @ digit_inline_rewrite) @ tld_rewrite @ add_space_before_dot) + + delete_space + + pynutil.delete('"') + ).optimize() + + # 6) protocol (like “https://” or “file:///”) + protocol = ( + pynutil.delete('protocol: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + insert_space + ) + + protocol_raw = pynutil.delete('protocol: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + cc_protocol_guard = pynini.accep("신용카드") + pynini.closure(NEMO_NOT_QUOTE, 0) + cc_protocol = (protocol_raw @ cc_protocol_guard) + insert_space + + # Credit card case: "신용카드 ..." protocol + 16-digit domain grouped as 4-4-4-4 + cc_graph = ( + cc_protocol + + delete_space + + pynutil.delete("domain:") + + delete_space + + pynutil.delete('"') + + cc_domain + + pynutil.delete('"') + + delete_space + ).optimize() + + # 7) Combine: optional protocol + optional username + domain + default_graph = ( + pynini.closure(protocol + delete_space, 0, 1) + + pynini.closure(user_name + delete_space + pynutil.insert(" 골뱅이 ") + delete_space, 0, 1) + + domain + + delete_space + ).optimize() + + graph = (cc_graph | default_graph) @ pynini.cdrewrite(delete_extra_space, "", "", NEMO_SIGMA) + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py new file mode 100644 index 000000000..d523dba00 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -0,0 +1,163 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing Korean fractions, e.g. + tokens { fraction { numerator: "3" denominator: "5" } } → 5분의3 + tokens { fraction { integer_part: "2" numerator: "7" denominator: "9" } } → 2과 9분의7 + tokens { fraction { denominator: "√8" numerator: "4" } } → 루트8분의4 + tokens { fraction { denominator: "2.75" numerator: "125" } } → 2.75분의125 + tokens { fraction { negative: "마이너스" numerator: "10" denominator: "11" } } → 마이너스11분의10 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + # Handles square root symbols like "√3" → "루트3" + denominator_root = pynini.cross("√", "루트") + pynutil.insert(NEMO_SPACE) + pynini.closure(NEMO_NOT_QUOTE) + numerator_root = pynini.cross("√", "루트") + pynutil.insert(NEMO_SPACE) + pynini.closure(NEMO_NOT_QUOTE) + + # Matches non-root numeric content + denominator = pynini.closure(NEMO_NOT_QUOTE - "√") + numerator = pynini.closure(NEMO_NOT_QUOTE - "√") + + # Delete FST field: denominator and extract value + denominator_component = ( + pynutil.delete('denominator: "') + (denominator_root | denominator) + pynutil.delete('"') + ) + numerator_component = pynutil.delete('numerator: "') + (numerator_root | numerator) + pynutil.delete('"') + + # Match fraction form: "denominator + 분의 + numerator" + # Also deletes optional morphosyntactic_features: "분의" if present + graph_fraction = ( + denominator_component + + pynutil.delete(NEMO_SPACE) + + pynini.closure( + pynutil.delete('morphosyntactic_features:') + delete_space + pynutil.delete('"분의"') + delete_space, + 0, + 1, + ) + + pynutil.insert("분의") + + pynutil.insert(NEMO_SPACE) + + numerator_component + ) + + # Handle subject particle feature (분의_subject) + # Insert default particle "이" (will be corrected later via rewrite rules) + subject_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_subject"') + + delete_space + + pynutil.insert("이") # 일단 기본값 + ) + + # Handle topic particle feature (분의_topic) + topic_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_topic"') + + delete_space + + pynutil.insert("은") + ) + + # Handle object particle feature (분의_object) + object_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_object"') + + delete_space + + pynutil.insert("을") + ) + + # Combine fraction + optional particle suffix + # Particle is always inserted first in default form and later corrected + graph_fraction_all = graph_fraction + pynini.closure(subject_suffix | topic_suffix | object_suffix, 0, 1) + + # Handle integer + fraction (e.g., "2과 3/4") + # integer_part is removed and replaced with proper spacing + graph_integer = ( + pynutil.delete('integer_part:') + + delete_space + + pynutil.delete('"') + + pynini.closure(pynini.union("√", ".", NEMO_NOT_QUOTE - '"')) + + pynutil.delete('"') + + pynutil.insert(NEMO_SPACE) + ) + # Combine integer part with fraction + graph_integer_fraction = graph_integer + delete_space + graph_fraction_all + + # Handle optional negative prefix (e.g., "마이너스") + optional_sign = ( + pynutil.delete('negative:') + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE - '"') + + pynutil.delete('"') + + delete_space + + pynutil.insert(NEMO_SPACE) + ) + + # Final structure: + # [optional negative] + (integer + fraction OR fraction only) + graph = pynini.closure(optional_sign, 0, 1) + (graph_integer_fraction | graph_fraction_all) + + # Remove token wrappers + final_graph = self.delete_tokens(graph) + + # Sigma for rewrite context (entire string) + sigma = pynini.closure(NEMO_NOT_QUOTE | NEMO_SPACE) + + # Fix particle agreement for vowel-ending numerals. + # Subject: 이 -> 가 + # Topic: 은 -> 는 + # Object: 을 -> 를 + particle_rewrite = pynini.cdrewrite( + pynini.string_map( + [ + # Subject particle + ("이이", "이가"), + ("사이", "사가"), + ("오이", "오가"), + ("구이", "구가"), + # Topic particle + ("이은", "이는"), + ("사은", "사는"), + ("오은", "오는"), + ("구은", "구는"), + # Object particle + ("이을", "이를"), + ("사을", "사를"), + ("오을", "오를"), + ("구을", "구를"), + ] + ), + "", + "", + sigma, + ) + + # Apply all rewrite rules sequentially and final optimized FST + final_graph = final_graph @ particle_rewrite + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/measure.py b/nemo_text_processing/text_normalization/ko/verbalizers/measure.py new file mode 100644 index 000000000..765b143cd --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/measure.py @@ -0,0 +1,89 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing Korean measure tokens into surface text. + measure { cardinal { integer: "<...>" } units: "<...>" } + + Converts tokens like: + measure { cardinal { integer: "이" } units: "킬로그램" } + measure { fraction { numerator: "이" denominator: "삼" } units: "킬로미터" } + + into surface text: + "이 킬로그램", "삼분의 이 킬로미터" + + Args: + decimal, cardinal, fraction: Sub-verbalizers handling number types. + deterministic: If True, outputs a single normalized form. + """ + + def __init__( + self, + decimal: GraphFst = None, + cardinal: GraphFst = None, + fraction: GraphFst = None, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + # Combine all numeric verbalizers + graph_cardinal = cardinal.fst + graph_decimal = decimal.fst + graph_fraction = fraction.fst + + # Add a space after "마이너스" if it appears within numeric blocks + minus_space_rewrite = pynini.cdrewrite(pynini.cross("마이너스", "마이너스 "), "", "", NEMO_SIGMA).optimize() + + # Apply rewrite to each numeric subgraph to ensure spacing after "마이너스" + cardinal_spaced = graph_cardinal @ minus_space_rewrite + fraction_spaced = graph_fraction @ minus_space_rewrite + decimal_spaced = graph_decimal @ minus_space_rewrite + + # Combine all supported numeric types (cardinal | decimal | fraction) + number_block = decimal_spaced | cardinal_spaced | fraction_spaced + + # Extract and output unit string + units = ( + delete_space + + pynutil.delete("units:") + + delete_space + + pynutil.delete('"') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + # Normal form: + main = number_block + insert_space + units + + # preserve_order form: + preserve_order = delete_space + pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + alt = units + insert_space + number_block + pynini.closure(preserve_order) + + graph = main | alt + + # Merge and clean tokens + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/money.py b/nemo_text_processing/text_normalization/ko/verbalizers/money.py new file mode 100644 index 000000000..333f68011 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/money.py @@ -0,0 +1,110 @@ +# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + +# ===== whitespace & token helpers ===== +sp = pynini.closure(delete_space) # absorb 0+ spaces +FIELD_VAL = pynini.closure(NEMO_NOT_QUOTE, 1) + + +def del_key_val(key: str): + """ + Delete the token field prefix and quotes, keep only the value. + + Input format: [sp] key: "" + Output: + + Example: + input 'integer_part: "삼백오십"' + output '삼백오십' + """ + return (sp + pynutil.delete(f'{key}: "') + FIELD_VAL + pynutil.delete('"')).optimize() + + +def drop_key_val(key: str): + """ + Delete the entire key-value pair (key and its quoted value). + + Input format: [sp] key: "" + Output: (nothing) + + Example: + input 'minor_part: "십"' + output '' + """ + return (sp + pynutil.delete(f'{key}: "') + FIELD_VAL + pynutil.delete('"')).optimize() + + +def drop_key_exact(key: str, val: str): + """ + Delete the exact key-value pair if it matches the given value. + + Input format: [sp] key: "val" + Output: (nothing) + + Example: + input 'currency_maj: "원"' + output '' + """ + return (sp + pynutil.delete(f'{key}: "{val}"')).optimize() + + +class MoneyFst(GraphFst): + """ + Verbalize Korean money. + + Input tokens: + tokens { money { integer_part: "..." currency_maj: "..." [minor_part: "..."] } } + + Period (e.g., /월, /년, …) is intentionally NOT handled here. + Output examples: + integer_part: "십" currency_maj: "원" -> "십원" + integer_part: "삼십억" currency_maj: "원" -> "삼십억원" + integer_part: "이백" currency_maj: "달러" -> "이백 달러" + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + # --- fields --- + integer_part = del_key_val("integer_part") + minor_part_drop = drop_key_val("minor_part") # ignore minor for KRW + currency_val_any = del_key_val("currency_maj") # ex) "원", "달러", "유로" + won_key_drop = drop_key_exact("currency_maj", "원") # don't print the key for KRW + + # ===== KRW (원) ===== + # (A) [integer] [원] -> "{integer}원" + won_a = integer_part + sp + won_key_drop + pynutil.insert("원") + # (B) [원] [integer] -> "{integer}원" + won_b = won_key_drop + sp + integer_part + pynutil.insert("원") + won_core = won_a | won_b + won_core = (won_core + pynini.closure(minor_part_drop, 0, 1)).optimize() + + # ===== Other currencies ===== + # "{integer} {currency}" (KRW sticks; others are spaced) + other_core = (integer_part + insert_space + currency_val_any).optimize() + other_core = (other_core + pynini.closure(minor_part_drop, 0, 1)).optimize() + + # ===== combine (no period) ===== + graph_core = (pynutil.add_weight(won_core, 0.0) | pynutil.add_weight(other_core, 0.5)).optimize() + + # no trailing period mapping + graph = graph_core + + # strip tokens wrapper + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/ko/verbalizers/ordinal.py new file mode 100644 index 000000000..c8c06a0c4 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing korean ordinal e.g. + tokens { ordinal { integer: "1번째" } } -> 첫번째 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + graph_integer = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + + final_graph = graph_integer + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/telephone.py b/nemo_text_processing/text_normalization/ko/verbalizers/telephone.py new file mode 100644 index 000000000..211358141 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/telephone.py @@ -0,0 +1,83 @@ +# Copyright (c) 2025 NVIDIA CORPORATION.  All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#     http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing Korean telephone numbers. + + Input: + telephone { [country_code: "...",] number_part: "..." [extension: "..."] } + Output: + [country_code + " "] + number_part [+ ", 내선 " + extension] + + Examples: + telephone { country_code: "국가번호 팔이," number_part: "영일영 삼칠일삼 칠영오영" } + -> 플러스 팔 이, 영일영, 삼칠일삼, 칠영오영 + telephone { number_part: "팔영영 오오오 영영영영" extension: "이삼사" } + -> 팔영영, 오오오, 영영영영, 내선 이삼사 + + Args: + deterministic: if True provides a single transduction; if False allows multiple. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + # country_code (optional, add trailing space if present) + country = ( + pynini.closure(delete_space, 0, 1) + + pynutil.delete('country_code: "') + + pynutil.insert("국가번호 ") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + insert_space + ) + + # number_part (mandatory) + number = ( + pynini.closure(delete_space, 0, 1) + + pynutil.delete('number_part: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + # extension (optional, prepend with ", 내선 ") + ext_field = ( + pynini.closure(delete_space, 0, 1) + + pynutil.delete('extension: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + extension_opt = pynini.closure(pynutil.insert(", 내선 ") + ext_field, 0, 1) + + # remove wrapper "telephone { ... }" + graph = ( + pynutil.delete("telephone") + + pynini.closure(delete_space, 0, 1) + + pynutil.delete("{") + + pynini.closure(country, 0, 1) + + number + + extension_opt + + pynini.closure(delete_space, 0, 1) + + pynutil.delete("}") + ) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/time.py b/nemo_text_processing/text_normalization/ko/verbalizers/time.py new file mode 100644 index 000000000..4d1414b1e --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/time.py @@ -0,0 +1,63 @@ +# Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing time e.g. + + + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + hour_component = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + minute_content = pynini.closure(NEMO_NOT_QUOTE) + minute_component = pynutil.delete("minutes: \"") + pynini.cross("영분", "") + pynutil.delete( + "\"" + ) | pynutil.delete("minutes: \"") + (minute_content - "영분") + pynutil.delete("\"") + + second_content = pynini.closure(NEMO_NOT_QUOTE) + second_component = pynutil.delete("seconds: \"") + pynini.cross("영초", "") + pynutil.delete( + "\"" + ) | pynutil.delete("seconds: \"") + (second_content - "영초") + pynutil.delete("\"") + + division_component = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_basic_time = pynini.closure(division_component + delete_space + insert_space, 0, 1) + ( + ( + hour_component + + delete_space + + insert_space + + minute_component + + delete_space + + insert_space + + second_component + ) + | (hour_component + delete_space + insert_space + minute_component) + | hour_component + | minute_component + | second_component + ) + + final_graph = graph_basic_time + + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py new file mode 100644 index 000000000..d3d5b951a --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py @@ -0,0 +1,74 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ko.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ko.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.ko.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ko.verbalizers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.ko.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.ko.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.ko.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.ko.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ko.verbalizers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.ko.verbalizers.time import TimeFst +from nemo_text_processing.text_normalization.ko.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.ko.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + date = DateFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + decimal = DecimalFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + fraction = FractionFst(deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + money = MoneyFst(deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + electronic = ElectronicFst(deterministic=deterministic) + + graph = pynini.union( + cardinal.fst, + ordinal.fst, + word.fst, + decimal.fst, + fraction.fst, + date.fst, + whitelist.fst, + time.fst, + money.fst, + telephone.fst, + measure.fst, + electronic.fst, + ) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py new file mode 100644 index 000000000..0271a4b7b --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py @@ -0,0 +1,72 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.ko.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ko_tn_{deterministic}_verbalizer.far") + + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + token_graph = VerbalizeFst(deterministic=deterministic) + + token_verbalizer = ( + pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") + ) + + space_between_tokens = pynini.closure(NEMO_WHITE_SPACE, 1) + + verbalizer = ( + delete_space + + token_verbalizer + + pynini.closure(space_between_tokens + token_verbalizer) + + delete_space + ) + + self.fst = verbalizer.optimize() + + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/ko/verbalizers/whitelist.py new file mode 100644 index 000000000..786b3afbf --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/whitelist.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WhiteListFst(GraphFst): + """ + tokens { name: "부터" } -> 부터 + tokens { name: "~" } -> ~ + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR - " ", 1) + + pynutil.delete("\"") + ) + + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/word.py b/nemo_text_processing/text_normalization/ko/verbalizers/word.py new file mode 100644 index 000000000..a14abd553 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/word.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WordFst(GraphFst): + """ + Korean verbalizer for word. + tokens { name: "이름" } -> 이름 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + + graph = pynutil.delete("name: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index 4db992c21..f4641f1cb 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -29,6 +29,10 @@ import regex import tqdm from joblib import Parallel, delayed +from pynini.lib.rewrite import top_rewrite +from sacremoses import MosesDetokenizer +from tqdm import tqdm + from nemo_text_processing.text_normalization.data_loader_utils import ( load_file, post_process_punct, @@ -37,9 +41,7 @@ ) from nemo_text_processing.text_normalization.preprocessing_utils import additional_split from nemo_text_processing.text_normalization.token_parser import PRESERVE_ORDER_KEY, TokenParser -from pynini.lib.rewrite import top_rewrite -from sacremoses import MosesDetokenizer -from tqdm import tqdm +from nemo_text_processing.utils.logging import logger # this is to handle long input sys.setrecursionlimit(3000) @@ -96,6 +98,7 @@ class Normalizer: Note: punct_post_process flag in normalize() supports all languages. max_number_of_permutations_per_split: a maximum number of permutations which can be generated from input sequence of tokens. + verbose: whether to print intermediate meta information """ def __init__( @@ -115,8 +118,8 @@ def __init__( self.post_processor = None if lang == "en": - from nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst from nemo_text_processing.text_normalization.en.verbalizers.post_processing import PostProcessingFst + from nemo_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst if post_process: self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) @@ -159,9 +162,38 @@ def __init__( elif lang == 'ar': from nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ar.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'hi': + from nemo_text_processing.text_normalization.hi.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.hi.verbalizers.post_processing import PostProcessingFst + from nemo_text_processing.text_normalization.hi.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) elif lang == 'it': from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'hy': + from nemo_text_processing.text_normalization.hy.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.hy.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'rw': + from nemo_text_processing.text_normalization.rw.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.rw.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'ja': + from nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'vi': + from nemo_text_processing.text_normalization.vi.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.vi.verbalizers.post_processing import PostProcessingFst + from nemo_text_processing.text_normalization.vi.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + elif lang == 'pt': + from nemo_text_processing.text_normalization.pt.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.pt.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'ko': + from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -315,15 +347,16 @@ def normalize( Args: text: string that may include semiotic classes - verbose: whether to print intermediate meta information punct_pre_process: whether to perform punctuation pre-processing, for example, [25] -> [ 25 ] punct_post_process: whether to normalize punctuation + verbose: whether to print intermediate meta information Returns: spoken form """ + logger.setLevel('DEBUG' if verbose else 'INFO') if len(text.split()) > 500: - print( - "WARNING! Your input is too long and could take a long time to normalize." + logger.warning( + "Your input is too long and could take a long time to normalize. " "Use split_text_into_sentences() to make the input shorter and then call normalize_list()." ) original_text = text @@ -331,33 +364,37 @@ def normalize( text = pre_process(text) text = text.strip() if not text: - if verbose: - print(text) + logger.debug(text) return text text = pynini.escape(text) tagged_lattice = self.find_tags(text) tagged_text = Normalizer.select_tag(tagged_lattice) - if verbose: - print(tagged_text) + logger.debug(tagged_text) + self.parser(tagged_text) tokens = self.parser.parse() split_tokens = self._split_tokens_to_reduce_number_of_permutations(tokens) output = "" for s in split_tokens: - tags_reordered = self.generate_permutations(s) - verbalizer_lattice = None - for tagged_text in tags_reordered: - tagged_text = pynini.escape(tagged_text) - - verbalizer_lattice = self.find_verbalizer(tagged_text) - if verbalizer_lattice.num_states() != 0: - break - if verbalizer_lattice is None: - raise ValueError(f"No permutations were generated from tokens {s}") - output += ' ' + Normalizer.select_verbalizer(verbalizer_lattice) + try: + tags_reordered = self.generate_permutations(s) + verbalizer_lattice = None + for tagged_text in tags_reordered: + tagged_text = pynini.escape(tagged_text) + + verbalizer_lattice = self.find_verbalizer(tagged_text) + if verbalizer_lattice.num_states() != 0: + break + if verbalizer_lattice is None: + logger.warning(f"No permutations were generated from tokens {s}") + return text + output += ' ' + Normalizer.select_verbalizer(verbalizer_lattice) + except Exception as e: + logger.warning("Failed text: " + text + str(e)) + return text output = SPACE_DUP.sub(' ', output[1:]) - if self.lang == "en" and hasattr(self, 'post_processor'): + if self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None: output = self.post_process(output) if punct_post_process: @@ -408,6 +445,7 @@ def normalize_manifest( batch_size: int, output_filename: Optional[str] = None, text_field: str = "text", + verbose: bool = False, **kwargs, ): """ @@ -430,6 +468,7 @@ def _process_batch( batch_idx: int, batch: List[str], dir_name: str, + verbose=verbose, punct_pre_process=False, punct_post_process=True, text_field: str = "text", @@ -446,7 +485,7 @@ def _process_batch( normalized_lines = [ self.normalize_line( line=line, - verbose=False, + verbose=verbose, punct_post_process=punct_post_process, punct_pre_process=punct_pre_process, text_field=text_field, @@ -458,9 +497,14 @@ def _process_batch( with open(f"{dir_name}/{batch_idx:06}.json", "w") as f_out: for line in normalized_lines: + if isinstance(line[output_field], set): + if len(line[output_field]) > 1: + logger.warning("Len of " + str(line[output_field]) + " > 1 ") + line[output_field] = line[output_field].pop() + f_out.write(json.dumps(line, ensure_ascii=False) + '\n') - print(f"Batch -- {batch_idx} -- is complete") + logger.info(f"Batch -- {batch_idx} -- is complete") if output_filename is None: output_filename = manifest.replace('.json', '_normalized.json') @@ -468,7 +512,7 @@ def _process_batch( with open(manifest, 'r') as f: lines = f.readlines() - print(f'Normalizing {len(lines)} line(s) of {manifest}...') + logger.warning(f'Normalizing {len(lines)} line(s) of {manifest}...') # to save intermediate results to a file batch = min(len(lines), batch_size) @@ -484,6 +528,7 @@ def _process_batch( lines[i : i + batch], tmp_dir, text_field=text_field, + verbose=verbose, punct_pre_process=punct_pre_process, punct_post_process=punct_post_process, **kwargs, @@ -498,10 +543,10 @@ def _process_batch( lines = f_in.read() f_out.write(lines) - print(f'Normalized version saved at {output_filename}') + logger.warning(f'Normalized version saved at {output_filename}') def split_text_into_sentences(self, text: str, additional_split_symbols: str = "") -> List[str]: - """ + r""" Split text into sentences. Args: @@ -515,8 +560,8 @@ def split_text_into_sentences(self, text: str, additional_split_symbols: str = " upper_case_unicode = "" if self.lang == "ru": - lower_case_unicode = '\u0430-\u04FF' - upper_case_unicode = '\u0410-\u042F' + lower_case_unicode = '\u0430-\u04ff' + upper_case_unicode = '\u0410-\u042f' # end of quoted speech - to be able to split sentences by full stop text = re.sub(r"([\.\?\!])([\"\'])", r"\g<2>\g<1> ", text) @@ -560,7 +605,7 @@ def _permute(self, d: OrderedDict) -> List[str]: elif isinstance(v, bool): subl = ["".join(x) for x in itertools.product(subl, [f"{k}: true "])] else: - raise ValueError() + raise ValueError("Key: " + str(k) + " Value: " + str(v)) l.extend(subl) return l @@ -639,6 +684,7 @@ def select_verbalizer(lattice: 'pynini.FstLike') -> str: Args: lattice: verbalization lattice + text: full text line to raise in case of an exception Returns: shortest path """ @@ -694,7 +740,7 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "fr", "hu", "se", "sv", "zh", "ar", "it"], + choices=["en", "de", "es", "fr", "hu", "se", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"], default="en", type=str, ) @@ -707,6 +753,11 @@ def parse_args(): type=str, ) parser.add_argument("--verbose", help="print info for debugging", action='store_true') + parser.add_argument( + "--no_post_process", + help="WFST-based post processing, e.g. to remove extra spaces added during TN, normalize punctuation marks [could differ from the input]. Only Eng is supported, not supported in Sparrowhawk", + action="store_true", + ) parser.add_argument( "--punct_post_process", help="Add this flag to enable punctuation post processing to match input.", @@ -751,6 +802,7 @@ def parse_args(): normalizer = Normalizer( input_case=args.input_case, + post_process=not args.no_post_process, cache_dir=args.cache_dir, overwrite_cache=args.overwrite_cache, whitelist=whitelist, @@ -759,14 +811,15 @@ def parse_args(): ) start_time = perf_counter() if args.input_string: - print( - normalizer.normalize( - args.input_string, - verbose=args.verbose, - punct_pre_process=args.punct_pre_process, - punct_post_process=args.punct_post_process, - ) + output = normalizer.normalize( + args.input_string, + verbose=args.verbose, + punct_pre_process=args.punct_pre_process, + punct_post_process=args.punct_post_process, ) + print("=" * 40) + print(output) + print("=" * 40) elif args.input_file: if args.input_file.endswith(".json"): normalizer.normalize_manifest( @@ -778,13 +831,14 @@ def parse_args(): text_field=args.manifest_text_field, output_field=args.output_field, output_filename=args.output_file, + verbose=args.verbose, ) else: - print("Loading data: " + args.input_file) + logger.info("Loading data: " + args.input_file) data = load_file(args.input_file) - print("- Data: " + str(len(data)) + " sentences") + logger.info("- Data: " + str(len(data)) + " sentences") normalizer_prediction = normalizer.normalize_list( data, verbose=args.verbose, @@ -793,8 +847,8 @@ def parse_args(): ) if args.output_file: write_file(args.output_file, normalizer_prediction) - print(f"- Normalized. Writing out to {args.output_file}") + logger.info(f"- Normalized. Writing out to {args.output_file}") else: - print(normalizer_prediction) + logger.info(normalizer_prediction) - print(f"Execution time: {perf_counter() - start_time:.02f} sec") + logger.info(f"Execution time: {perf_counter() - start_time:.02f} sec") diff --git a/nemo_text_processing/text_normalization/normalize_with_audio.py b/nemo_text_processing/text_normalization/normalize_with_audio.py index a69493bbd..8a60516cc 100644 --- a/nemo_text_processing/text_normalization/normalize_with_audio.py +++ b/nemo_text_processing/text_normalization/normalize_with_audio.py @@ -20,11 +20,12 @@ import editdistance import pynini +from pynini.lib import rewrite + from nemo_text_processing.text_normalization.data_loader_utils import post_process_punct, pre_process from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.utils_audio_based import get_alignment -from pynini.lib import rewrite - +from nemo_text_processing.utils.logging import logger """ The script provides multiple normalization options and chooses the best one that minimizes CER of the ASR output @@ -163,16 +164,21 @@ def normalize( text_with_span_tags_list[masked_idx_list[sem_tag_idx]] = "" else: non_deter_options = self.normalize_non_deterministic( - text=cur_semiotic_span, n_tagged=n_tagged, punct_post_process=punct_post_process, verbose=verbose, + text=cur_semiotic_span, + n_tagged=n_tagged, + punct_post_process=punct_post_process, + verbose=verbose, ) try: best_option, cer, _ = self.select_best_match( - normalized_texts=non_deter_options, pred_text=cur_pred_text, verbose=verbose, + normalized_texts=non_deter_options, + pred_text=cur_pred_text, + verbose=verbose, ) if cer_threshold > 0 and cer > cer_threshold: best_option = cur_deter_norm - if verbose and True: - print( + if verbose: + logger.info( f"CER of the best normalization option is above cer_theshold, using determinictis option. CER: {cer}" ) except: @@ -202,7 +208,7 @@ def normalize_non_deterministic( text = text.strip() if not text: if verbose: - print(text) + logger.info(text) return text text = pynini.escape(text) @@ -237,7 +243,8 @@ def normalize_non_deterministic( self._verbalize(tagged_text, normalized_texts, n_tagged, verbose=verbose) if len(normalized_texts) == 0: - raise ValueError() + logger.warning("Failed text: " + text + ", normalized_texts: " + str(normalized_texts)) + return text if punct_post_process: # do post-processing based on Moses detokenizer @@ -289,7 +296,7 @@ def normalize_line( line = json.loads(line) normalized_text = self.normalize( - text=line["text"], + text=line[text_field], verbose=verbose, n_tagged=n_tagged, punct_post_process=punct_post_process, @@ -358,13 +365,17 @@ def get_verbalized_text(tagged_text): tagged_text_reordered = pynini.escape(tagged_text_reordered) normalized_texts.extend(get_verbalized_text(tagged_text_reordered)) if verbose: - print(tagged_text_reordered) + logger.info(tagged_text_reordered) except pynini.lib.rewrite.Error: continue def select_best_match( - self, normalized_texts: List[str], pred_text: str, verbose: bool = False, remove_punct: bool = False, + self, + normalized_texts: List[str], + pred_text: str, + verbose: bool = False, + remove_punct: bool = False, ): """ Selects the best normalization option based on the lowest CER @@ -383,10 +394,10 @@ def select_best_match( normalized_text, cer, idx = normalized_texts_cer[0] if verbose: - print('-' * 30) + logger.info('-' * 30) for option in normalized_texts: - print(option) - print('-' * 30) + logger.info(option) + logger.info('-' * 30) return normalized_text, cer, idx @@ -510,7 +521,7 @@ def parse_args(): verbose=args.verbose, ) for option in options: - print(option) + logger.info(option) elif args.manifest.endswith('.json'): normalizer = NormalizerWithAudio( input_case=args.input_case, @@ -532,10 +543,11 @@ def parse_args(): text_field=args.manifest_text_field, asr_pred_field=args.manifest_asr_pred_field, cer_threshold=args.cer_threshold, + verbose=args.verbose, ) else: raise ValueError( "Provide either path to .json manifest with '--manifest' OR " + "an input text with '--text' (for debugging without audio)" ) - print(f'Execution time: {round((perf_counter() - start)/60, 2)} min.') + logger.info(f'Execution time: {round((perf_counter() - start)/60, 2)} min.') diff --git a/nemo_text_processing/text_normalization/pt/__init__.py b/nemo_text_processing/text_normalization/pt/__init__.py new file mode 100644 index 000000000..ffd13e2d6 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/__init__.py b/nemo_text_processing/text_normalization/pt/data/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/date/__init__.py b/nemo_text_processing/text_normalization/pt/data/date/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/date/months.tsv b/nemo_text_processing/text_normalization/pt/data/date/months.tsv new file mode 100644 index 000000000..6713229bd --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/date/months.tsv @@ -0,0 +1,21 @@ +1 janeiro +01 janeiro +2 fevereiro +02 fevereiro +3 março +03 março +4 abril +04 abril +5 maio +05 maio +6 junho +06 junho +7 julho +07 julho +8 agosto +08 agosto +9 setembro +09 setembro +10 outubro +11 novembro +12 dezembro diff --git a/nemo_text_processing/text_normalization/pt/data/date/numeric_separators.tsv b/nemo_text_processing/text_normalization/pt/data/date/numeric_separators.tsv new file mode 100644 index 000000000..ee24567bc --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/date/numeric_separators.tsv @@ -0,0 +1,3 @@ +/ +. +- diff --git a/nemo_text_processing/text_normalization/pt/data/date/verbal_phrases.tsv b/nemo_text_processing/text_normalization/pt/data/date/verbal_phrases.tsv new file mode 100644 index 000000000..d04c0fa50 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/date/verbal_phrases.tsv @@ -0,0 +1 @@ +preposition de diff --git a/nemo_text_processing/text_normalization/pt/data/electronic/__init__.py b/nemo_text_processing/text_normalization/pt/data/electronic/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/electronic/domain.tsv b/nemo_text_processing/text_normalization/pt/data/electronic/domain.tsv new file mode 100644 index 000000000..b9daa19a5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/electronic/domain.tsv @@ -0,0 +1,7 @@ +.com ponto com +.com.br ponto com ponto br +.gov.br ponto gov ponto br +.org ponto org +.net ponto net +.edu ponto edu +.br ponto br diff --git a/nemo_text_processing/text_normalization/pt/data/electronic/electronic_spoken_unit.tsv b/nemo_text_processing/text_normalization/pt/data/electronic/electronic_spoken_unit.tsv new file mode 100644 index 000000000..698bc2773 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/electronic/electronic_spoken_unit.tsv @@ -0,0 +1,16 @@ +google google +usuario usuario +microsoft microsoft +amazon amazon +facebook facebook +meta meta +netflix netflix +spotify spotify +samsung samsung +apple apple +linkedin linkedin +instagram instagram +whatsapp whatsapp +oracle oracle +adobe adobe +paypal paypal diff --git a/nemo_text_processing/text_normalization/pt/data/electronic/server_name.tsv b/nemo_text_processing/text_normalization/pt/data/electronic/server_name.tsv new file mode 100644 index 000000000..50b4eeb65 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/electronic/server_name.tsv @@ -0,0 +1,6 @@ +gmail +nvidia +outlook +hotmail +yahoo +live diff --git a/nemo_text_processing/text_normalization/pt/data/electronic/symbols.tsv b/nemo_text_processing/text_normalization/pt/data/electronic/symbols.tsv new file mode 100644 index 000000000..85c8a1b10 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/electronic/symbols.tsv @@ -0,0 +1,22 @@ +. ponto +- traço +_ underscore +! exclamação +# cerquilha +$ dólar +% por cento +& e comercial +' apóstrofo +* asterisco ++ mais +/ barra += igual +? interrogação +^ acento circunflexo +` crase +{ chave esquerda +| barra vertical +} chave direita +~ til +, vírgula +: dois pontos diff --git a/nemo_text_processing/text_normalization/pt/data/fractions/__init__.py b/nemo_text_processing/text_normalization/pt/data/fractions/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/fractions/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/fractions/ordinal_exceptions.tsv b/nemo_text_processing/text_normalization/pt/data/fractions/ordinal_exceptions.tsv new file mode 100644 index 000000000..22c9ed16c --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/fractions/ordinal_exceptions.tsv @@ -0,0 +1,2 @@ +segundo meio +terceiro terço diff --git a/nemo_text_processing/text_normalization/pt/data/fractions/powers_of_ten.tsv b/nemo_text_processing/text_normalization/pt/data/fractions/powers_of_ten.tsv new file mode 100644 index 000000000..b19c44364 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/fractions/powers_of_ten.tsv @@ -0,0 +1 @@ +mil milésimo diff --git a/nemo_text_processing/text_normalization/pt/data/fractions/specials.tsv b/nemo_text_processing/text_normalization/pt/data/fractions/specials.tsv new file mode 100644 index 000000000..7efeba5e6 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/fractions/specials.tsv @@ -0,0 +1,5 @@ +connector e +minus menos +plural_suffix s +avos_suffix avos +avos_between diff --git a/nemo_text_processing/text_normalization/pt/data/measure/__init__.py b/nemo_text_processing/text_normalization/pt/data/measure/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/measure/measurements_plural.tsv b/nemo_text_processing/text_normalization/pt/data/measure/measurements_plural.tsv new file mode 100755 index 000000000..d9cbfb87f --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/measure/measurements_plural.tsv @@ -0,0 +1,64 @@ +h horas +min minutos +s segundos +ms milissegundos +ns nanossegundos +μs microssegundos +t toneladas +kg quilos +kg quilogramas +g gramas +mg miligramas +μm micrômetros +nm nanômetros +mm milímetros +cm centímetros +cm² centímetros quadrados +cm³ centímetros cúbicos +m metros +m² metros quadrados +m³ metros cúbicos +km quilômetros +km² quilômetros quadrados +ha hectares +kph quilômetros por hora +mph milhas por hora +m/s metros por segundo +l litros +ml mililitros +kgf quilogramas força +% por cento +°F fahrenheit +°F graus fahrenheit +°C graus celsius +Hz hertz +kHz quilo hertz +MHz mega hertz +GHz giga hertz +W watts +kW quilowatts +MW megawatts +GW gigawatts +Wh watts hora +kWh quilowatts hora +MWh megawatts hora +GWh gigawatts hora +kV quilovolts +V volts +mV milivolts +A amperes +mA miliamperes +rpm rotações por minuto +db decibéis +cal calorias +kcal quilocalorias +G gramas +KG quilos +KG quilogramas +KM quilômetros +M metros +L litros +ML mililitros +M2 metros quadrados +M^2 metros quadrados +C graus celsius diff --git a/nemo_text_processing/text_normalization/pt/data/measure/measurements_singular.tsv b/nemo_text_processing/text_normalization/pt/data/measure/measurements_singular.tsv new file mode 100755 index 000000000..242aba14e --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/measure/measurements_singular.tsv @@ -0,0 +1,65 @@ +h hora +min minuto +s segundo +ms milissegundo +ns nanossegundo +μs microssegundo +t tonelada +kg quilo +kg quilograma +g grama +mg miligrama +μm micrômetro +nm nanômetro +mm milímetro +cm centímetro +cm² centímetro quadrado +cm³ centímetro cúbico +m metro +m² metro quadrado +m³ metro cúbico +km quilômetro +km² quilômetro quadrado +ha hectare +kph quilômetro por hora +mph milha por hora +m/s metro por segundo +l litro +ml mililitro +kgf quilograma força +% por cento +°F fahrenheit +°C celsius +°F grau fahrenheit +°C grau celsius +Hz hertz +kHz quilo hertz +MHz mega hertz +GHz giga hertz +W watt +kW quilowatt +MW megawatt +GW gigawatt +Wh watt hora +kWh quilowatt hora +MWh megawatt hora +GWh gigawatt hora +kV quilovolt +V volt +mV milivolt +A ampere +mA miliampere +rpm rotação por minuto +db decibel +cal caloria +kcal quilocaloria +G grama +KG quilo +KG quilograma +KM quilômetro +M metro +L litro +ML mililitro +M2 metro quadrado +M^2 metro quadrado +C celsius diff --git a/nemo_text_processing/text_normalization/pt/data/money/__init__.py b/nemo_text_processing/text_normalization/pt/data/money/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/pt/data/money/currency_major.tsv new file mode 100644 index 000000000..b23d1bcce --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/money/currency_major.tsv @@ -0,0 +1,5 @@ +US$ dólar americano +R$ real +€ euro +£ libra esterlina +$ dólar diff --git a/nemo_text_processing/text_normalization/pt/data/money/currency_major_plural.tsv b/nemo_text_processing/text_normalization/pt/data/money/currency_major_plural.tsv new file mode 100644 index 000000000..feca270ff --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/money/currency_major_plural.tsv @@ -0,0 +1,5 @@ +dólar americano dólares americanos +real reais +euro euros +libra esterlina libras esterlinas +dólar dólares diff --git a/nemo_text_processing/text_normalization/pt/data/money/currency_minor.tsv b/nemo_text_processing/text_normalization/pt/data/money/currency_minor.tsv new file mode 100644 index 000000000..4e9a95d66 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/money/currency_minor.tsv @@ -0,0 +1,5 @@ +US$ centavo +R$ centavo +€ centavo +£ centavo +$ centavo diff --git a/nemo_text_processing/text_normalization/pt/data/money/currency_minor_plural.tsv b/nemo_text_processing/text_normalization/pt/data/money/currency_minor_plural.tsv new file mode 100644 index 000000000..2ac61dff7 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/money/currency_minor_plural.tsv @@ -0,0 +1 @@ +centavo centavos diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/__init__.py b/nemo_text_processing/text_normalization/pt/data/numbers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/cardinal_specials.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/cardinal_specials.tsv new file mode 100644 index 000000000..04ea91ee4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/cardinal_specials.tsv @@ -0,0 +1,4 @@ +connector e +thousand mil +hundred_100 cem +hundred_1 cento diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/decimal_fractional_specials.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/decimal_fractional_specials.tsv new file mode 100644 index 000000000..c84a95f53 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/decimal_fractional_specials.tsv @@ -0,0 +1,3 @@ +001 mil e um +010 mil e dez +100 mil e cem diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/decimal_specials.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/decimal_specials.tsv new file mode 100644 index 000000000..f6257d9d1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/decimal_specials.tsv @@ -0,0 +1,2 @@ +separator vírgula +minus menos diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/digit.tsv new file mode 100644 index 000000000..1859416c8 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 um +2 dois +3 três +4 quatro +5 cinco +6 seis +7 sete +8 oito +9 nove diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/hundreds.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/hundreds.tsv new file mode 100644 index 000000000..620f512b3 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/hundreds.tsv @@ -0,0 +1,8 @@ +2 duzentos +3 trezentos +4 quatrocentos +5 quinhentos +6 seiscentos +7 setecentos +8 oitocentos +9 novecentos diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/quantity_words.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/quantity_words.tsv new file mode 100644 index 000000000..a94cbd553 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/quantity_words.tsv @@ -0,0 +1,9 @@ +mil +milhão +milhões +bilhão +bilhões +trilhão +trilhões +quatrilhão +quatrilhões diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/scales.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/scales.tsv new file mode 100644 index 000000000..a3dffe4e0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/scales.tsv @@ -0,0 +1,4 @@ +one_label plural_suffix magnitude_zeros +um milhão milhões 0 +um bilhão bilhões 9 +um trilhão trilhões 12 diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/teens.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/teens.tsv new file mode 100644 index 000000000..50c4e0b8d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/teens.tsv @@ -0,0 +1,10 @@ +10 dez +11 onze +12 doze +13 treze +14 catorze +15 quinze +16 dezesseis +17 dezessete +18 dezoito +19 dezenove diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/tens.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/tens.tsv new file mode 100644 index 000000000..43c4a8bc6 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/tens.tsv @@ -0,0 +1,8 @@ +2 vinte +3 trinta +4 quarenta +5 cinquenta +6 sessenta +7 setenta +8 oitenta +9 noventa diff --git a/nemo_text_processing/text_normalization/pt/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/pt/data/numbers/zero.tsv new file mode 100644 index 000000000..29be0f38b --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 zero diff --git a/nemo_text_processing/text_normalization/pt/data/ordinals/__init__.py b/nemo_text_processing/text_normalization/pt/data/ordinals/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/ordinals/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/ordinals/digit.tsv b/nemo_text_processing/text_normalization/pt/data/ordinals/digit.tsv new file mode 100644 index 000000000..5fefbc3b8 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/ordinals/digit.tsv @@ -0,0 +1,10 @@ +primeiro um +segundo dois +terceiro três +quarto quatro +quinto cinco +sexto seis +sétimo sete +oitavo oito +nono nove +décimo dez diff --git a/nemo_text_processing/text_normalization/pt/data/ordinals/feminine.tsv b/nemo_text_processing/text_normalization/pt/data/ordinals/feminine.tsv new file mode 100644 index 000000000..c75ae5ed0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/ordinals/feminine.tsv @@ -0,0 +1,11 @@ +primeiro primeira +segundo segunda +terceiro terceira +quarto quarta +quinto quinta +sexto sexta +sétimo sétima +oitavo oitava +nono nona +décimo décima +ésimo ésima diff --git a/nemo_text_processing/text_normalization/pt/data/ordinals/hundreds.tsv b/nemo_text_processing/text_normalization/pt/data/ordinals/hundreds.tsv new file mode 100644 index 000000000..6d919a86c --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/ordinals/hundreds.tsv @@ -0,0 +1,10 @@ +centésimo cem +centésimo cento +ducentésimo duzentos +trecentésimo trezentos +quadringentésimo quatrocentos +quincentésimo quinhentos +sexcentésimo seiscentos +septingentésimo setecentos +octingentésimo oitocentos +noningentésimo novecentos diff --git a/nemo_text_processing/text_normalization/pt/data/ordinals/specials.tsv b/nemo_text_processing/text_normalization/pt/data/ordinals/specials.tsv new file mode 100644 index 000000000..bb6933fe6 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/ordinals/specials.tsv @@ -0,0 +1,2 @@ +connector_in e +connector_out diff --git a/nemo_text_processing/text_normalization/pt/data/ordinals/teen.tsv b/nemo_text_processing/text_normalization/pt/data/ordinals/teen.tsv new file mode 100644 index 000000000..1b1e191c9 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/ordinals/teen.tsv @@ -0,0 +1,9 @@ +décimo primeiro onze +décimo segundo doze +décimo terceiro treze +décimo quarto catorze +décimo quinto quinze +décimo sexto dezesseis +décimo sétimo dezessete +décimo oitavo dezoito +décimo nono dezenove diff --git a/nemo_text_processing/text_normalization/pt/data/ordinals/ties.tsv b/nemo_text_processing/text_normalization/pt/data/ordinals/ties.tsv new file mode 100644 index 000000000..f40700034 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/ordinals/ties.tsv @@ -0,0 +1,8 @@ +vigésimo vinte +trigésimo trinta +quadragésimo quarenta +quinquagésimo cinquenta +sexagésimo sessenta +septuagésimo setenta +octogésimo oitenta +nonagésimo noventa diff --git a/nemo_text_processing/text_normalization/pt/data/telephone/__init__.py b/nemo_text_processing/text_normalization/pt/data/telephone/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/telephone/ip_prompt.tsv b/nemo_text_processing/text_normalization/pt/data/telephone/ip_prompt.tsv new file mode 100644 index 000000000..7d59e35eb --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/telephone/ip_prompt.tsv @@ -0,0 +1,2 @@ +ip ip +endereço de ip endereço de i p diff --git a/nemo_text_processing/text_normalization/pt/data/telephone/telephone_prompt.tsv b/nemo_text_processing/text_normalization/pt/data/telephone/telephone_prompt.tsv new file mode 100644 index 000000000..62efeccd5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/telephone/telephone_prompt.tsv @@ -0,0 +1,4 @@ +ligue para +telefone +celular +meu número é diff --git a/nemo_text_processing/text_normalization/pt/data/time/__init__.py b/nemo_text_processing/text_normalization/pt/data/time/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/time/day_period_suffix.tsv b/nemo_text_processing/text_normalization/pt/data/time/day_period_suffix.tsv new file mode 100644 index 000000000..a942795f4 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/time/day_period_suffix.tsv @@ -0,0 +1,4 @@ +manhã da manhã 6 11 +tarde da tarde 12 17 +noite da noite 18 23 +madrugada da madrugada 0 5 diff --git a/nemo_text_processing/text_normalization/pt/data/whitelist/__init__.py b/nemo_text_processing/text_normalization/pt/data/whitelist/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/whitelist/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/data/whitelist/ipa_symbols.tsv b/nemo_text_processing/text_normalization/pt/data/whitelist/ipa_symbols.tsv new file mode 100644 index 000000000..f5559c711 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/whitelist/ipa_symbols.tsv @@ -0,0 +1,521 @@ +a +aoj +aəj +aː +aːʲ +aː͡j +aː͡ɨ̯ +aˤ +aˤː +a̠ +a̠ː +a̰ +a͡e +a͡i +a͡iː +a͡i̯ +a͡j +a͡o +a͡u +a͡uː +a͡u̯ +a͡w +a͡ə +a͡ɨ̯ +a͡ɪ +a͡ʊ +b +bʱ +bʲ +bː +b̥ +c +cʰ +cː +ç +d +dʲ +dː +d̥ +d̪ +d̪ʱ +d͡z +d͡zʷ +d͡zː +d͡ʑ +d͡ʒ +d͡ʒʱ +d͡ʒʲ +d͡ʒː +e +eː +eːʲ +eː͡j +ẽː +ẽ͡j̃ +e̞ +e̞ː +e̯ +e͡i +e͡iː +e͡ɨ̯ +f +fʲ +fː +h +hː +i +iəj +iəw +iʲ +iː +iːʲ +ĩː +i̥ +i̯ +i͡u +i͡ə +i͡ɛ +j +jː +j̃ +k +kʰ +kʰː +kʲ +kʲʼ +kʷ +kʷʼ +kʼ +kː +k̚ +k̚ʲ +k̟̚ +k͈ +k͡p̚ +l +lʲ +lː +l̥ +l̩ +m +mʲ +mʲː +mː +m̥ +m̩ +n +nʲ +nː +n̥ +n̩ +o +oʲ +oː +oːʲ +ò +õ͡j̃ +õ͡w̃ +o̝ +o̞ +o̞ː +o̯ +o̰ +o͡u +o͡uː +p +pʰ +pʰː +pʲ +pʷʼ +pʼ +pː +p̚ +p̚ʲ +p͈ +p͜f +p͡f +q +qʷ +qʼ +r +rʲ +rː +r̂ +r̂ː +r̥ +r̩ +s +sʰ +sʲ +sʼ +sː +s͈ +t +tʰ +tʰː +tʲ +tʷʼ +tʼ +tː +t̚ +t̪ +t̪ʰ +t͈ +t͜s +t͡s +t͡sʰ +t͡sʰː +t͡sʲ +t͡sʷ +t͡sʼ +t͡sː +t͡ɕ +t͡ɕʰ +t͡ɕ͈ +t͡ʂ +t͡ʂʼ +t͡ʃ +t͡ʃʰ +t͡ʃʰː +t͡ʃʲ +t͡ʃʷ +t͡ʃʼ +t͡ʃː +u +uəj +uʲ +uː +uːʲ +ũː +ũ͡j̃ +u̯ +u͡e +u͡i +u͡j +u͡ɔ +u͡ə +v +vʲ +vː +w +w̃ +x +xʷ +xː +y +yː +yːʲ +y̯ +z +zʲ +zː +z̥ +à +àː +á +áː +â +âː +ã +ã̠ +æ +æː +æ̀ +æ̀ː +æ̂ +æ̂ː +æ͡ɪ +æ͡ʉ +ç +è +èː +é +éː +ê +êː +ì +ìː +í +íː +î +îː +ï +ð +ò +òː +ó +óː +ô +ôː +õ +õː +õ̞ +ø +øː +øːʲ +ø̯ +ù +ùː +ú +úː +û +ûː +ā +āː +ē +ēː +ĕ +ĕ͡ə +ě +ěː +ħ +ĩ +ĩː +ī +īː +ŋ +ŋʲ +ŋ̊ +ŋ̍ +ŋ̟ +ŋ̩ +ŋ͡m +ō +ŏ +ŏ͡ə +œ +œː +œ̃ +œ͡i +œ͡iː +œ͡ʏ +ř +řː +ũ +ũː +ū +ūː +ŭ +ŭ͡ə +ǎ +ǎː +ǐ +ǐː +ǒ +ǒː +ǔ +ǔː +ǣ +ǣː +ɐ +ɐː +ɐ̃ +ɐ̃͡j̃ +ɐ̃͡w̃ +ɐ̯ +ɐ̯̯ +ɑ +ɑː +ɑ̃ +ɑ̃ː +ɒ +ɒʲ +ɒː +ɓ +ɔ +ɔː +ɔˤː +ɔ̀ +ɔ̀ː +ɔ́ +ɔ́ː +ɔ̃ +ɔ̃ː +ɔ̰ +ɔ͡i̯ +ɔ͡ə +ɔ͡ɨ̯ +ɔ͡ɪ +ɔ͡ʊ +ɕ +ɕʰ +ɕː +ɕ͈ +ɖ +ɖʱ +ɗ +ɘ +ɘː +ə +əː +əˤ +ə̀ +ə́ +ə̃ +ə̯ +ə͡u̯ +ə͡w +ə͡ɨ +ə͡ɨ̯ +ɚ +ɛ +ɛʲ +ɛː +ɛˤː +ɛ̀ +ɛ̀ː +ɛ́ +ɛ́ː +ɛ̂ +ɛ̂ː +ɛ̃ +ɛ̃ː +ɛ̄ +ɛ̄ː +ɛ̰ +ɛ͡i +ɛ͡i̯ +ɛ͡u +ɛ͡u̯ +ɛ͡ɪ +ɛ͡ʊ +ɜ +ɜː +ɝ +ɝː +ɟ +ɟː +ɟ͡ʝ +ɡ +ɡʱ +ɡʲ +ɡʷ +ɡː +ɡ̊ +ɣ +ɤ +ɥ +ɦ +ɨ +ɨəj +ɨː +ɨ̃ᵝ +ɨ̞ +ɨ̥ᵝ +ɨ̯ +ɨ͡u̯ +ɨ͡w +ɨ͡ə +ɨᵝ +ɨᵝː +ɪ +ɪː +ɪ̀ +ɪ́ +ɪ̃ +ɪ̯ +ɪ̰ +ɪ͡u̯ +ɪ͡ʊ +ɫ +ɫː +ɬ +ɬʼ +ɭ +ɮ +ɯ +ɯː +ɯ̟̃ᵝ +ɯ̟̊ᵝ +ɯ̟ᵝ +ɯ̟ᵝː +ɰ +ɰ̃ +ɰᵝ +ɱ +ɱ̩ +ɲ +ɲː +ɲ̊ +ɲ̟ +ɳ +ɴ +ɸ +ɸʷ +ɹ +ɻ +ɽ +ɽʱ +ɾ +ɾʲ +ɾː +ɾ̝̊ +ʀ +ʁ +ʁʷ +ʁː +ʂ +ʂʷ +ʃ +ʃʰ +ʃʲ +ʃʷ +ʃʷʼ +ʃʼ +ʃː +ʈ +ʈʰ +ʉ +ʉː +ʊ +ʊ̀ +ʊ́ +ʊ̃ +ʊ̯ +ʊ̯͡i +ʊ̯͡ɨ +ʊ̰ +ʋ +ʌ +ʌ̹ +ʍ +ʎ +ʏ +ʏː +ʏ̯ +ʐ +ʐʷ +ʑ +ʒ +ʒʲ +ʒʷ +ʒː +ʔ +ʔʲ +ʔʷ +ʝ +˦ˀ˥ +˦˥ +˦˧˥ +˦˩ +˧ˀ˨ +˧˦ +˧˧ +˧˨ +˧˩ +˨˩ +˨˩˦ +˨˩˨ +β +θ +χ +χʷ +χː +ḛ +ḭ +ṵ +ẽ +ẽː +ẽ̞ +‿ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/pt/data/whitelist/symbol.tsv b/nemo_text_processing/text_normalization/pt/data/whitelist/symbol.tsv new file mode 100644 index 000000000..7f7b525e3 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/data/whitelist/symbol.tsv @@ -0,0 +1,23 @@ +& e comercial +# cerquilha +@ arroba +§ parágrafo +™ marca comercial +® marca registrada +© direitos autorais +_ sublinhado +% por cento +* asterisco ++ mais +/ barra += igual +^ acento circunflexo +| barra vertical +~ til +$ dólar +£ libra esterlina +€ euro +₩ won +¥ ienes +° grau +º ordinal masculino diff --git a/nemo_text_processing/text_normalization/pt/data/whitelist/tts.tsv b/nemo_text_processing/text_normalization/pt/data/whitelist/tts.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/nemo_text_processing/text_normalization/pt/graph_utils.py b/nemo_text_processing/text_normalization/pt/graph_utils.py new file mode 100644 index 000000000..1d548deb5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/graph_utils.py @@ -0,0 +1,193 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Portuguese (PT) text normalization graph utilities. + +Self-contained module with no dependency on en.graph_utils. Provides character/digit +symbols (NEMO_*), space helpers (delete_space, insert_space, delete_extra_space), +GraphFst base class, generator_main for FAR export, and PT-specific helpers +(filter_cardinal_punctuation, shift_cardinal_gender_pt). +""" + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.utils.logging import logger + +# ---- Character/digit symbols (same semantics as EN) ---- +NEMO_CHAR = utf8.VALID_UTF8_CHAR +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_SPACE = " " +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, pynini.accep('"')).optimize() +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() + +MIN_NEG_WEIGHT = -0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ").optimize() + +delete_preserve_order = pynini.closure(pynutil.delete(" preserve_order: true")) + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]) -> None: + """ + Export one or more graphs to an OpenFst Finite State Archive (FAR) file. + + Args: + file_name: path to the output .far file. + graphs: mapping of rule names to FST graphs to export. + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +class GraphFst: + """ + Base class for all Portuguese text normalization grammar FSTs. + + Args: + name: name of the grammar (e.g. "cardinal", "decimal"). + kind: either "classify" or "verbalize". + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization). + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(os.path.abspath(__file__)) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) + + +# ---- PT-specific (Brazilian: 1.000.000 or 1 000 000) ---- +cardinal_separator = pynini.string_map([".", " "]) + + +def filter_cardinal_punctuation(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Parse digit groups separated by cardinal_separator (e.g. 1.000.000) then apply fst. + + Args: + fst: FST that maps digit string to verbalized cardinal. + + Returns: + Composed FST that accepts digit strings with optional thousand separators. + """ + exactly_three = NEMO_DIGIT**3 + up_to_three = pynini.closure(NEMO_DIGIT, 1, 3) + cardinal_string = pynini.closure(NEMO_DIGIT, 1) + cardinal_string |= ( + up_to_three + + pynutil.delete(cardinal_separator) + + pynini.closure(exactly_three + pynutil.delete(cardinal_separator)) + + exactly_three + ) + return cardinal_string @ fst + + +def shift_cardinal_gender_pt(fst: "pynini.FstLike") -> "pynini.FstLike": + """ + Apply Portuguese masculine-to-feminine conversion for cardinal strings, e.g. + "um" -> "uma", "dois" -> "duas", "duzentos" -> "duzentas". + + Args: + fst: FST producing masculine cardinal verbalization. + + Returns: + FST that produces feminine form when composed with the same input. + """ + fem_ones = pynini.cdrewrite( + pynini.cross("um", "uma"), + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) + fem_twos = pynini.cdrewrite( + pynini.cross("dois", "duas"), + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) + fem_hundreds = pynini.cdrewrite( + pynini.cross("entos", "entas"), + pynini.union("duz", "trez", "quatroc", "quinh", "seisc", "setec", "oitoc", "novec"), + pynini.union(NEMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), + NEMO_SIGMA, + ) + return fst @ fem_ones @ fem_twos @ fem_hundreds + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/pt/taggers/__init__.py b/nemo_text_processing/text_normalization/pt/taggers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/taggers/cardinal.py b/nemo_text_processing/text_normalization/pt/taggers/cardinal.py new file mode 100644 index 000000000..0f14f3b46 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/cardinal.py @@ -0,0 +1,303 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from functools import reduce + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + filter_cardinal_punctuation, + insert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying Portuguese cardinals, e.g. + "1000" -> cardinal { integer: "mil" } + "2.000.000" -> cardinal { integer: "dois milhões" } + "-5" -> cardinal { negative: "true" integer: "cinco" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + specials = { + row[0]: row[1] for row in load_labels(get_abs_path("data/numbers/cardinal_specials.tsv")) if len(row) >= 2 + } + connector_e = insert_space + pynutil.insert(specials["connector"]) + insert_space + thousand = specials["thousand"] + hundred_100 = specials["hundred_100"] + hundred_1 = specials["hundred_1"] + + scale_rows = load_labels(get_abs_path("data/numbers/scales.tsv")) + scales = [(row[0], row[1], int(row[2])) for row in scale_rows if len(row) >= 3 and row[2].strip().isdigit()] + + _num = lambda p: pynini.string_file(get_abs_path(f"data/numbers/{p}")) + zero, digit, teens, tens, hundreds = ( + _num("zero.tsv"), + _num("digit.tsv"), + _num("teens.tsv"), + _num("tens.tsv"), + _num("hundreds.tsv"), + ) + digits_no_one = (NEMO_DIGIT - "1") @ digit + + graph_tens = teens | (tens + (pynutil.delete("0") | (connector_e + digit))) + self.tens = graph_tens.optimize() + self.two_digit_non_zero = pynini.union(digit, graph_tens, (pynini.cross("0", NEMO_SPACE) + digit)).optimize() + + # After "X00" hundreds (oitocentos, …), suffix "01"-"09" needs leading zero stripped + # (graph_tens has no path for "09"; connector+digit only consumes one digit). + graph_hundreds = hundreds + pynini.union( + pynutil.delete("00"), + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + (connector_e + digit), + ) + # "100" -> cem only (cross("1", cento)+delete("00") would also match "100" but + # yields "cento"; OpenFst vs pynini top_rewrite can disagree on ties — Sparrowhawk). + graph_hundreds |= pynini.cross("100", hundred_100) + graph_hundreds |= pynini.cross("1", hundred_1) + pynini.union( + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + ) + self.hundreds = graph_hundreds.optimize() + + h_comp_base = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens) + h_comp = h_comp_base | (pynutil.delete("00") + digit) + h_comp_no_one = h_comp_base | (pynutil.delete("00") + digits_no_one) + + pure_tens_input = pynini.union(*[pynini.accep(str(d * 10)) for d in range(1, 10)]) + graph_pure_tens_only = pure_tens_input @ graph_tens + graph_compound_tens = (pynini.closure(NEMO_DIGIT, 2, 2) - pure_tens_input) @ graph_tens + + graph_pure_components = pynini.union( + pynutil.delete("0") + graph_pure_tens_only, + pynutil.delete("00") + digit, + hundreds + pynutil.delete("00"), + pynini.cross("100", hundred_100), + ) + graph_compound_hundreds = pynini.union( + pynini.cross("1", hundred_1) + + pynini.union( + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + ), + hundreds + + pynini.union( + (connector_e + graph_tens), + (connector_e + pynutil.delete("0") + digit), + (connector_e + digit), + ), + ) + + suffix_after_mil = pynini.union( + pynutil.delete("000"), + (connector_e + graph_pure_components), + (insert_space + graph_compound_hundreds), + # Use connector_e so "2024" -> dois mil e vinte e quatro (not dois mil vinte e quatro). + (connector_e + pynutil.delete("0") + graph_compound_tens), + ) + + t_comp = pynini.union( + pynutil.delete("000") + h_comp, + h_comp_no_one + insert_space + pynutil.insert(thousand) + suffix_after_mil, + pynini.cross("001", thousand) + suffix_after_mil, + ) + t_comp_no_one = pynini.union( + pynutil.delete("000") + h_comp_no_one, + h_comp_no_one + + insert_space + + pynutil.insert(thousand) + + ((insert_space + h_comp) | pynutil.delete("000")), + pynini.cross("001", thousand) + ((insert_space + h_comp) | pynutil.delete("000")), + ) + + graph_large_scales = pynini.accep("") + for one_label, plural_suffix, _ in reversed(scales): + g = pynutil.add_weight(pynini.cross("000001", one_label), -0.001) + g |= t_comp_no_one + pynutil.insert(plural_suffix) + g |= pynutil.delete("000000") + g += insert_space + graph_large_scales += g + + # 9/12-digit: scale block + trailing (million+thousands, billion+9digits) + scale_3_mil = self._scale_block_3(scales[0][0], scales[0][1], h_comp_no_one) + scale_3_bi = self._scale_block_3(scales[1][0], scales[1][1], h_comp_no_one) + graph_9 = self._build_scale_trailing_graph(scale_3_mil, t_comp, 6, 9) + graph_12 = self._build_scale_trailing_graph(scale_3_bi, graph_9, 9, 12) + pure_9, pure_12 = self._pure_inputs(9), self._pure_inputs(12) + trail_9 = (pure_9 @ graph_9, (NEMO_DIGIT**9 - pure_9) @ graph_9) + trail_12 = (pure_12 @ graph_12, (NEMO_DIGIT**12 - pure_12) @ graph_12) + + # Units 6 (u6): pure get "e" after scale; compound no "e" + u6_one = pynini.cross("000001", "1") @ digit + u6_pure = pynini.union( + u6_one, + pynini.cross("001000", thousand), + pynini.cross("000010", "10") @ graph_tens, + pynini.cross("000100", hundred_100), + (pynini.cross("010000", "10") @ graph_tens) + insert_space + pynutil.insert(thousand), + pynini.cross("100000", hundred_100) + insert_space + pynutil.insert(thousand), + ) + u6_compound = (NEMO_DIGIT**6 - self._pure_inputs(6)) @ t_comp + u6 = u6_pure | u6_compound + z18 = pynini.accep("0" * 18) # 18 zeros: branch no "e" + smaller_e = (connector_e + u6_pure) | u6_compound | pynutil.delete("0" * 6) + smaller = u6 | pynutil.delete("0" * 6) + graph_24 = (((NEMO_DIGIT**18 - z18) + NEMO_DIGIT**6) @ (graph_large_scales + smaller_e)) | ( + (z18 + NEMO_DIGIT**6) @ (pynutil.delete(z18) + smaller) + ) + + trail_by_z = {9: trail_9, 12: trail_12} + magnitude_patterns = [ + self._build_magnitude_pattern( + one_label, + plural_suffix, + magnitude_zeros, + trail_by_z.get(magnitude_zeros), + connector_e, + insert_space, + digit, + graph_tens, + graph_hundreds, + ) + for one_label, plural_suffix, magnitude_zeros in scales + if magnitude_zeros > 0 + ] + + pad = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0) + pad = pad @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) @ NEMO_DIGIT**24 + norm = pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite( + delete_space, "", "[EOS]", NEMO_SIGMA + ) + norm = norm @ pynini.cdrewrite( + pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 2), NEMO_SPACE), NEMO_ALPHA, NEMO_ALPHA, NEMO_SIGMA + ) + self.graph = reduce(lambda a, b: a | b, magnitude_patterns, pad @ graph_24 @ norm) | zero + self.graph = filter_cardinal_punctuation(self.graph).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() + + def _scale_block_3(self, one_label, plural_suffix, component_no_one): + """001->one_label, 000->'', else component+plural.""" + return pynini.union( + pynini.cross("001", one_label), + pynini.cross("000", ""), + (NEMO_DIGIT**3 - pynini.accep("001") - pynini.accep("000")) + @ (component_no_one + insert_space + pynutil.insert(plural_suffix)), + ) + + def _build_scale_trailing_graph(self, scale_3, sub_graph, trailing_len, total_len): + """total_len digits = scale_3 + trailing; no trailing space when trailing all zeros.""" + zt, ztotal = "0" * trailing_len, "0" * total_len + scale_nonzero = NEMO_DIGIT**3 - pynini.accep("000") + branches = [ + (pynini.accep("000") + NEMO_DIGIT**trailing_len) @ (pynutil.delete("000") + sub_graph), + (scale_nonzero + (NEMO_DIGIT**trailing_len - pynini.accep(zt))) @ (scale_3 + insert_space + sub_graph), + (scale_nonzero + pynini.accep(zt)) @ (scale_3 + pynutil.delete(zt)), + (pynini.accep("000") + pynini.accep(zt)) @ pynutil.delete(ztotal), + ] + return pynini.union(*branches) + + @staticmethod + def _pure_inputs(num_digits): + """Inputs 1, 10, 100, ... as num_digits-digit strings.""" + return pynini.union(*[pynini.accep(str(10**k).zfill(num_digits)) for k in range(0, num_digits)]) + + def _magnitude_graph( + self, + one_word, + plural_suffix, + zero_count, + graph_digit, + graph_tens, + graph_hundreds, + connector_e, + insert_space, + trailing_pair=None, + ): + """Round (1–3 digit + scale + zeros); optional trailing (e + pure | space + compound).""" + zeros = "0" * zero_count + round_pats = [] + trail_pats = [] if trailing_pair else None + for L in (1, 2, 3): + total = zero_count + L + if L == 1: + lead = pynini.cross("1", one_word) | ((NEMO_DIGIT - "1") @ graph_digit + pynutil.insert(plural_suffix)) + else: + lead = pynini.closure(NEMO_DIGIT, L, L) @ (graph_tens if L == 2 else graph_hundreds) + pynutil.insert( + plural_suffix + ) + lead_fst = NEMO_DIGIT**L @ lead + round_pats.append(pynini.closure(NEMO_DIGIT, total, total) @ (lead_fst + pynutil.delete(zeros))) + if trailing_pair: + pure, compound = trailing_pair + trail_part = NEMO_DIGIT**zero_count @ (connector_e + pure) | NEMO_DIGIT**zero_count @ ( + insert_space + compound + ) + trail_pats.append(pynini.closure(NEMO_DIGIT, total, total) @ (lead_fst + trail_part)) + graph_round = pynini.union(*round_pats) + graph_trail = pynini.union(*trail_pats) if trail_pats else None + return graph_round, graph_trail + + def _build_magnitude_pattern( + self, + one_label, + plural_suffix, + magnitude_zeros, + trailing_pair, + connector_e, + insert_space, + graph_digit, + graph_tens, + graph_hundreds, + ): + """Restrict length; round + optional non-zero trailing.""" + restrict = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, magnitude_zeros, magnitude_zeros + 2) + graph_round, graph_trail = self._magnitude_graph( + one_label, + plural_suffix, + magnitude_zeros, + graph_digit, + graph_tens, + graph_hundreds, + connector_e, + insert_space, + trailing_pair, + ) + if graph_trail is None: + return pynutil.add_weight(restrict @ graph_round, -1.0) + non_zero_trail = pynini.union( + *[NEMO_DIGIT**n + (NEMO_DIGIT**magnitude_zeros - pynini.accep("0" * magnitude_zeros)) for n in (1, 2, 3)] + ) + return pynutil.add_weight(restrict @ (graph_round | (non_zero_trail @ graph_trail)), -1.0) diff --git a/nemo_text_processing/text_normalization/pt/taggers/date.py b/nemo_text_processing/text_normalization/pt/taggers/date.py new file mode 100644 index 000000000..8f12677ab --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/date.py @@ -0,0 +1,151 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying Portuguese (Brazilian) dates, e.g. + 15/03/2024 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + 15 de março de 2024 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + 2024-03-15 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + 03/15/2024 -> date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + numbers = cardinal.graph + + month_rows = load_labels(get_abs_path("data/date/months.tsv")) + month_pairs = [(r[0], r[1]) for r in month_rows if len(r) >= 2] + month_to_word = pynini.string_map(month_pairs).optimize() + + day_10_31 = ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ pynini.union(*[str(x) for x in range(10, 32)]) @ numbers + day_02_09 = pynutil.delete("0") + (pynini.union(*[str(x) for x in range(2, 10)]) @ numbers) + day_2_9 = pynini.union(*[str(x) for x in range(2, 10)]) @ numbers + day_inner = pynini.union( + pynini.cross("01", "primeiro"), + day_10_31, + day_02_09, + day_2_9, + pynini.cross("1", "primeiro"), + ).optimize() + day_part = pynutil.insert('day: "') + day_inner + pynutil.insert('"') + + month_digits = ( + pynini.union("10", "11", "12") + | pynutil.delete("0") + pynini.union(*[str(x) for x in range(1, 10)]) + | pynini.union(*[str(x) for x in range(1, 10)]) + ) + month_num = month_digits @ month_to_word + month_part = pynutil.insert('month: "') + month_num + pynutil.insert('"') + + year_num = ((NEMO_DIGIT - "0") + NEMO_DIGIT**3) @ numbers + year_part = pynutil.insert('year: "') + year_num + pynutil.insert('"') + + preserve = pynutil.insert(" preserve_order: true") + + delete_de = delete_space + pynutil.delete("de") + delete_space + month_names = sorted({r[1] for r in month_rows if len(r) >= 2}, key=len, reverse=True) + text_pairs = [] + for name in month_names: + text_pairs.append((name, name)) + if name and name[0].islower(): + text_pairs.append((name[0].upper() + name[1:], name)) + month_written = pynutil.insert('month: "') + pynini.string_map(text_pairs).optimize() + pynutil.insert('"') + graph_text = day_part + delete_de + month_written + delete_de + year_part + preserve + + sep_path = get_abs_path("data/date/numeric_separators.tsv") + separators = [r[0].strip() for r in load_labels(sep_path) if r and r[0].strip()] + + one_or_two_digits = pynini.closure(NEMO_DIGIT, 1, 2) + year_four = (NEMO_DIGIT - "0") + NEMO_DIGIT**3 + _mdy_weight = 0.05 + + months_spoken = sorted({r[1] for r in month_rows if len(r) >= 2}) + day_spokens = set() + for n in range(1, 32): + for key in (str(n), f"{n:02d}"): + dstr = pynini.shortestpath(pynini.compose(pynini.accep(key), day_inner.optimize())).string() + day_spokens.add(dstr) + + _preserve_tail = " preserve_order: true" + + ymd_to_dmy_graph = None + mdy_to_dmy_graph = None + for month in months_spoken: + for day in day_spokens: + # After year: + sigma (year value + quotes), delete month/day and trailing preserve + # so the input is fully consumed (mdy_to_dmy does not need this: sigma eats the tail). + ymd_curr = ( + pynutil.insert('day: "' + day + '" month: "' + month + '" ') + + pynini.accep("year:") + + NEMO_SIGMA + + pynutil.delete(' month: "' + month + '" day: "' + day + '"' + _preserve_tail) + ) + ymd_to_dmy_graph = ymd_curr if ymd_to_dmy_graph is None else pynini.union(ymd_to_dmy_graph, ymd_curr) + + mdy_curr = ( + pynutil.insert('day: "' + day + '" month: "' + month + '" ') + + pynutil.delete('month: "' + month + '" day: "' + day + '" ') + + pynini.accep("year:") + + NEMO_SIGMA + ) + mdy_to_dmy_graph = mdy_curr if mdy_to_dmy_graph is None else pynini.union(mdy_to_dmy_graph, mdy_curr) + + ymd_to_dmy_graph = ymd_to_dmy_graph.optimize() + mdy_to_dmy_graph = mdy_to_dmy_graph.optimize() + + patterns = [graph_text] + for sep in separators: + sep_accep = pynini.accep(pynini.escape(sep)) + del_sep = pynutil.delete(sep_accep) + + dmy_core = day_part + del_sep + insert_space + month_part + del_sep + insert_space + year_part + preserve + iso_core = year_part + del_sep + insert_space + month_part + del_sep + insert_space + day_part + preserve + mdy_core = month_part + del_sep + insert_space + day_part + del_sep + insert_space + year_part + preserve + + lhs_dmy = one_or_two_digits + sep_accep + one_or_two_digits + sep_accep + year_four + lhs_iso = year_four + sep_accep + one_or_two_digits + sep_accep + one_or_two_digits + lhs_mdy = one_or_two_digits + sep_accep + one_or_two_digits + sep_accep + year_four + + patterns.append(pynini.compose(lhs_dmy, dmy_core)) + patterns.append( + pynutil.add_weight( + pynini.compose( + pynini.compose(lhs_mdy, mdy_core), + mdy_to_dmy_graph, + ), + _mdy_weight, + ) + ) + patterns.append( + pynini.compose( + pynini.compose(lhs_iso, iso_core), + ymd_to_dmy_graph, + ) + ) + + self.fst = self.add_tokens(pynini.union(*patterns).optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/decimal.py b/nemo_text_processing/text_normalization/pt/taggers/decimal.py new file mode 100644 index 000000000..d9d5d6094 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/decimal.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_DIGIT, GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying Portuguese decimal numbers, e.g. + "1,26" -> decimal { integer_part: "um" fractional_part: "dois seis" } + "0,01" -> decimal { integer_part: "zero" fractional_part: "zero um" } + "-1,26" -> decimal { negative: "true" ... } + "1,33 milhões" / "1 milhão" -> decimal { ... quantity: "milhões" / "milhão" } + + The fractional mantissa (after the comma) is always read digit-by-digit (0–9), + including leading zeros. Integer part and quantities still use cardinals. + + Args: + cardinal: CardinalFst instance for integer verbalization in tags. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + _num = lambda name: pynini.string_file(get_abs_path(f"data/numbers/{name}")).optimize() + + comma = pynutil.delete(",") + quantity_words = _num("quantity_words.tsv") + digit = _num("digit.tsv") + zero = _num("zero.tsv") + graph_digit_or_zero = pynini.union(digit, zero) + digit_by_digit = (graph_digit_or_zero + pynini.closure(insert_space + graph_digit_or_zero)).optimize() + + fractional_digits = pynini.closure(NEMO_DIGIT, 1, 15) + graph_fractional = ( + pynutil.insert('fractional_part: "') + (fractional_digits @ digit_by_digit) + pynutil.insert('"') + ) + + non_zero_lead = pynini.difference(NEMO_DIGIT, pynini.accep("0")) + + graph_integer_zero = ( + pynutil.insert('integer_part: "') + pynini.cross("0", "zero") + pynutil.insert('"') + insert_space + ) + decimal_when_zero = graph_integer_zero + comma + insert_space + graph_fractional + + graph_integer_pos = ( + pynutil.insert('integer_part: "') + + (non_zero_lead + pynini.closure(NEMO_DIGIT, 0, 11)) @ cardinal_graph + + pynutil.insert('"') + + insert_space + ) + decimal_when_pos = graph_integer_pos + comma + insert_space + graph_fractional + + decimal_core = pynini.union(decimal_when_zero, decimal_when_pos) + integer_quantity = ( + pynutil.insert('integer_part: "') + + (pynini.closure(NEMO_DIGIT, 1, 12) @ cardinal_graph) + + pynutil.insert('"') + + insert_space + + delete_space + + pynutil.insert('quantity: "') + + quantity_words + + pynutil.insert('"') + ) + decimal_quantity = ( + decimal_core + delete_space + pynutil.insert('quantity: "') + quantity_words + pynutil.insert('"') + ) + final_graph_wo_sign = pynini.union(decimal_core, integer_quantity, decimal_quantity) + self.final_graph_wo_negative = final_graph_wo_sign.optimize() + optional_minus = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + final_graph = optional_minus + final_graph_wo_sign + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/electronic.py b/nemo_text_processing/text_normalization/pt/taggers/electronic.py new file mode 100644 index 000000000..ddd89f4e0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/electronic.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_ALPHA, NEMO_DIGIT, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying electronic strings in pt-BR: + abc@hotmail.com -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true } + https://www.abc.com -> electronic { protocol: "https://www." domain: "abc.com" preserve_order: true } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + full_stop = pynini.accep(".") + at_symbol = "@" + protocol_string = "protocol" + domain_string = "domain" + username_string = "username" + http = "http" + https = "https" + www = "www" + + symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] + symbols = pynini.union(*symbols) + symbols_no_full_stop = pynini.difference(symbols, full_stop) + accepted_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols_no_full_stop), 1) + all_characters = pynini.closure((NEMO_ALPHA | NEMO_DIGIT | symbols), 1) + + domain_component = full_stop + accepted_characters + domain_graph = ( + pynutil.insert(domain_string + ': "') + + (accepted_characters + pynini.closure(domain_component, 1)) + + pynutil.insert('"') + ) + + username = ( + pynutil.insert(username_string + ': "') + + all_characters + + pynutil.insert('"') + + pynini.cross(at_symbol, NEMO_SPACE) + ) + email = username + domain_graph + + social_tag = ( + pynini.cross(at_symbol, "") + + pynutil.insert(username_string + ': "') + + (accepted_characters | (accepted_characters + pynini.closure(domain_component, 1))) + + pynutil.insert('"') + ) + + protocol_start = pynini.accep(https + "://") | pynini.accep(http + "://") + protocol_end = pynini.accep(www + ".") + if not deterministic: + protocol_end |= pynini.cross(www + ".", "dáblio dáblio dáblio.") + + protocol = protocol_start | protocol_end | (protocol_start + protocol_end) + protocol = pynutil.insert(protocol_string + ': "') + protocol + pynutil.insert('"') + url = protocol + pynutil.insert(NEMO_SPACE) + domain_graph + + graph = url | domain_graph | email | social_tag + self.graph = graph + + final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true")) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/fraction.py b/nemo_text_processing/text_normalization/pt/taggers/fraction.py new file mode 100644 index 000000000..b5a206ff0 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/fraction.py @@ -0,0 +1,117 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_WHITE_SPACE, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying Portuguese fraction numbers, e.g. + "1/2" -> fraction { numerator: "um" denominator: "meio" morphosyntactic_features: "ordinal" } + "2 3/4" -> fraction { integer_part: "dois" numerator: "três" denominator: "quarto" ... } + "2/11" -> fraction { numerator: "dois" denominator: "onze" morphosyntactic_features: "avos" } + + Args: + cardinal: CardinalFst instance for number parts. + ordinal: OrdinalFst instance for denominator 2-10 and exceptions. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + # Denominators 2–10 use ordinal form (no data file: fixed set) + two_to_ten = pynini.union(*[pynini.accep(str(d)) for d in range(2, 11)]).optimize() + + ord_digit_rows = load_labels(get_abs_path("data/ordinals/digit.tsv")) + ordinal_digit = pynini.string_map([(r[1], r[0]) for r in ord_digit_rows if len(r) >= 2]).optimize() + + ord_exc_rows = load_labels(get_abs_path("data/fractions/ordinal_exceptions.tsv")) + ordinal_exceptions = pynini.string_map([(r[0], r[1]) for r in ord_exc_rows if len(r) >= 2]).optimize() + + ord_hundreds_rows = load_labels(get_abs_path("data/ordinals/hundreds.tsv")) + ordinal_hundreds = pynini.string_map([(r[1], r[0]) for r in ord_hundreds_rows if len(r) >= 2]).optimize() + + powers_rows = load_labels(get_abs_path("data/fractions/powers_of_ten.tsv")) + powers_of_ten = pynini.string_map([(r[0], r[1]) for r in powers_rows if len(r) >= 2]).optimize() + + denom_ordinal_form = two_to_ten @ cardinal_graph @ ordinal_digit + denom_ordinal_form = denom_ordinal_form @ pynini.cdrewrite(ordinal_exceptions, "", "", NEMO_SIGMA) + denom_ordinal = ( + pynutil.insert('denominator: "') + + denom_ordinal_form + + pynutil.insert('" morphosyntactic_features: "ordinal"') + ) + + denom_100 = ( + pynutil.insert('denominator: "') + + (pynini.accep("100") @ cardinal_graph @ ordinal_hundreds) + + pynutil.insert('" morphosyntactic_features: "ordinal"') + ) + denom_1000 = ( + pynutil.insert('denominator: "') + + (pynini.accep("1000") @ cardinal_graph @ powers_of_ten) + + pynutil.insert('" morphosyntactic_features: "ordinal"') + ) + + denom_ordinal_2_10_100_1000 = pynini.union(denom_ordinal, denom_100, denom_1000) + digit_plus = pynini.closure(NEMO_DIGIT, 1) + denom_avos_input = pynini.difference( + digit_plus, + pynini.union( + two_to_ten, + pynini.accep("100"), + pynini.accep("1000"), + ), + ) + denom_avos = ( + pynutil.insert('denominator: "') + + (denom_avos_input @ cardinal_graph) + + pynutil.insert('" morphosyntactic_features: "avos"') + ) + + denominator = pynini.union(denom_ordinal_2_10_100_1000, denom_avos) + + # Slash variants: ASCII /, Unicode ⁄ (U+2044), ∕ (U+2215); with or without spaces + slash_or_space_slash = pynini.union( + pynini.cross("/", '" '), + pynini.cross(" / ", '" '), + pynini.cross("\u2044", '" '), # fraction slash ⁄ + pynini.cross(" \u2044 ", '" '), + pynini.cross("\u2215", '" '), # division slash ∕ + pynini.cross(" \u2215 ", '" '), + ) + numerator = pynutil.insert('numerator: "') + cardinal_graph + slash_or_space_slash + fraction_core = numerator + denominator + + integer_part = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + insert_space + + optional_minus = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1) + + mixed = integer_part + pynini.closure(NEMO_WHITE_SPACE, 1) + fraction_core + graph = optional_minus + pynini.union(mixed, fraction_core) + + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/measure.py b/nemo_text_processing/text_normalization/pt/taggers/measure.py new file mode 100644 index 000000000..0906551d1 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/measure.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure (pt-BR), e.g. + 200 g -> measure { cardinal { integer: "duzentos" } units: "gramas" } + 1 kg -> measure { cardinal { integer: "um" } units: "quilo" } + 2,4 g -> measure { decimal { ... } units: "gramas" } + 1/2 l -> measure { fraction { ... } units: "litros" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + unit_singular = pynini.string_file(get_abs_path("data/measure/measurements_singular.tsv")) + unit_plural = pynini.string_file(get_abs_path("data/measure/measurements_plural.tsv")) + + graph_unit_singular = convert_space(unit_singular) + graph_unit_plural = convert_space(unit_plural) + + optional_graph_negative = pynini.closure(pynini.accep("-"), 0, 1) + + unit_plural = pynutil.insert('units: "') + graph_unit_plural + pynutil.insert('"') + unit_singular_graph = pynutil.insert('units: "') + graph_unit_singular + pynutil.insert('"') + + subgraph_decimal = decimal.fst + insert_space + pynini.closure(NEMO_SPACE, 0, 1) + unit_plural + + subgraph_cardinal = ( + (optional_graph_negative + (NEMO_SIGMA - "1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + unit_plural + ) + + subgraph_cardinal |= ( + (optional_graph_negative + pynini.accep("1")) @ cardinal.fst + + insert_space + + pynini.closure(delete_space, 0, 1) + + unit_singular_graph + ) + + subgraph_fraction = fraction.fst + insert_space + pynini.closure(delete_space, 0, 1) + unit_plural + + final_graph = subgraph_decimal | subgraph_cardinal | subgraph_fraction + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/money.py b/nemo_text_processing/text_normalization/pt/taggers/money.py new file mode 100644 index 000000000..e059891f5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/money.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money (pt-BR), e.g. + R$ 12 -> money { currency_maj: "reais" integer_part: "doze" } + R$ 12,05 -> money { currency_maj: "reais" integer_part: "doze" fractional_part: "cinco" + currency_min: "centavos" preserve_order: true } + R$ 0,20 -> money { fractional_part: "vinte" currency_min: "centavos" preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + decimal_separator = pynini.accep(",") + maj_singular = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) + maj_plural_map = pynini.string_file(get_abs_path("data/money/currency_major_plural.tsv")) + maj_plural_graph = maj_singular @ maj_plural_map + min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) + min_plural_map = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) + min_plural_graph = min_singular @ min_plural_map + + cardinal_graph = cardinal.graph + graph_decimal_final = decimal.final_graph_wo_negative + + graph_maj_singular = pynutil.insert('currency_maj: "') + maj_singular + pynutil.insert('"') + graph_maj_plural = pynutil.insert('currency_maj: "') + maj_plural_graph + pynutil.insert('"') + + graph_integer_one = ( + pynutil.insert('integer_part: "') + (pynini.accep("1") @ cardinal_graph) + pynutil.insert('"') + ) + + decimal_with_quantity = (NEMO_SIGMA + NEMO_ALPHA) @ graph_decimal_final + + graph_decimal_plural = pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + graph_decimal_plural = ((NEMO_SIGMA - "1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_plural + + graph_decimal_singular = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_decimal_final, + graph_decimal_final + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_decimal_singular = (pynini.accep("1") + decimal_separator + NEMO_SIGMA) @ graph_decimal_singular + + graph_decimal = pynini.union( + graph_decimal_singular, + graph_decimal_plural, + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + decimal_with_quantity, + ) + + graph_integer = pynutil.insert('integer_part: "') + ((NEMO_SIGMA - "1") @ cardinal_graph) + pynutil.insert('"') + + graph_integer_only = pynini.union( + graph_maj_singular + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer_one, + graph_integer_one + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_singular, + ) + graph_integer_only |= pynini.union( + graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer, + graph_integer + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural, + ) + + graph = graph_integer_only | graph_decimal + + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + + graph_min_singular = pynutil.insert('currency_min: "') + min_singular + pynutil.insert('"') + graph_min_plural = pynutil.insert('currency_min: "') + min_plural_graph + pynutil.insert('"') + + maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + decimal_graph_with_minor = None + for curr_symbol, _ in maj_singular_labels: + preserve_order = pynutil.insert(" preserve_order: true") + + integer_plus_maj = pynini.union( + graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural, + graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular, + ) + integer_plus_maj = (pynini.closure(NEMO_DIGIT) - "0") @ integer_plus_maj + + graph_fractional_one = ( + pynutil.insert('fractional_part: "') + + (two_digits_fractional_part @ pynini.cross("1", "um")) + + pynutil.insert('"') + ) + + graph_fractional = ( + two_digits_fractional_part @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") @ cardinal.two_digit_non_zero + ) + graph_fractional = pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"') + + fractional_plus_min = pynini.union( + graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural, + graph_fractional_one + insert_space + pynutil.insert(curr_symbol) @ graph_min_singular, + ) + + decimal_graph_with_minor_curr = ( + integer_plus_maj + pynini.cross(decimal_separator, NEMO_SPACE) + fractional_plus_min + ) + if not deterministic: + decimal_graph_with_minor_curr |= pynutil.add_weight( + integer_plus_maj + + pynini.cross(decimal_separator, NEMO_SPACE) + + pynutil.insert('fractional_part: "') + + two_digits_fractional_part @ cardinal.two_digit_non_zero + + pynutil.insert('"'), + weight=0.0001, + ) + + decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min + decimal_graph_with_minor_curr = pynini.union( + pynutil.delete(curr_symbol) + + pynini.closure(delete_space, 0, 1) + + decimal_graph_with_minor_curr + + preserve_order, + decimal_graph_with_minor_curr + + preserve_order + + pynini.closure(delete_space, 0, 1) + + pynutil.delete(curr_symbol), + ) + + decimal_graph_with_minor = ( + decimal_graph_with_minor_curr + if decimal_graph_with_minor is None + else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr) + ) + + final_graph = graph | pynutil.add_weight(decimal_graph_with_minor, -0.001) + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/ordinal.py b/nemo_text_processing/text_normalization/pt/taggers/ordinal.py new file mode 100644 index 000000000..e5ab95fa5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/ordinal.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying Portuguese ordinals, e.g. + "1º" / "1ª" -> ordinal { integer: "primeiro" / "primeira" morphosyntactic_features: "gender_masc" / "gender_fem" } + "21º" -> ordinal { integer: "vigésimo primeiro" morphosyntactic_features: "gender_masc" } + + Args: + cardinal: CardinalFst instance for composing compound ordinals. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph + + spec_rows = load_labels(get_abs_path("data/ordinals/specials.tsv")) + spec = {r[0]: r[1] for r in spec_rows if len(r) >= 2} + conn_in = spec.get("connector_in", " e ") + conn_out = spec.get("connector_out", NEMO_SPACE) + conn = pynini.cross(conn_in, conn_out) + + # Data: ordinal \t cardinal → FST cardinal→ordinal via load_labels + digit_rows = load_labels(get_abs_path("data/ordinals/digit.tsv")) + graph_digit = pynini.string_map([(r[1], r[0]) for r in digit_rows if len(r) >= 2]).optimize() + teen_rows = load_labels(get_abs_path("data/ordinals/teen.tsv")) + graph_teens = pynini.string_map([(r[1], r[0]) for r in teen_rows if len(r) >= 2]).optimize() + ties_rows = load_labels(get_abs_path("data/ordinals/ties.tsv")) + graph_ties = pynini.string_map([(r[1], r[0]) for r in ties_rows if len(r) >= 2]).optimize() + hundreds_rows = load_labels(get_abs_path("data/ordinals/hundreds.tsv")) + graph_hundreds = pynini.string_map([(r[1], r[0]) for r in hundreds_rows if len(r) >= 2]).optimize() + + graph_tens = pynini.union( + graph_teens, + graph_ties + pynini.closure(conn + graph_digit, 0, 1), + ) + graph_hundred_component = pynini.union( + graph_hundreds + pynini.closure(conn + pynini.union(graph_tens, graph_digit), 0, 1), + graph_tens, + graph_digit, + ) + ordinal_rewrite = graph_hundred_component.optimize() + ordinal_inner = cardinal_graph @ ordinal_rewrite + + opt_dot = pynini.closure(pynutil.delete("."), 0, 1) + suffix_masc = opt_dot + pynutil.delete(pynini.union("º", "°")) + suffix_fem = opt_dot + pynutil.delete("ª") + digit_block = pynini.closure(NEMO_DIGIT, 1, 3) + + to_ordinal_masc = (digit_block + suffix_masc) @ ordinal_inner + to_ordinal_fem = (digit_block + suffix_fem) @ ordinal_inner + + graph_masc = ( + pynutil.insert('integer: "') + + to_ordinal_masc + + pynutil.insert('" morphosyntactic_features: "gender_masc"') + ) + graph_fem = ( + pynutil.insert('integer: "') + to_ordinal_fem + pynutil.insert('" morphosyntactic_features: "gender_fem"') + ) + self.fst = self.add_tokens(pynini.union(graph_masc, graph_fem)).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/punctuation.py b/nemo_text_processing/text_normalization/pt/taggers/punctuation.py new file mode 100644 index 000000000..f42a8815f --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/punctuation.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation (pt-BR pipeline). + Mirrors the English punctuation tagger; whitelist symbols are excluded via data/whitelist/symbol.tsv. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + [r"\[", r"\]"] + + symbol_path = get_abs_path("data/whitelist/symbol.tsv") + whitelist_rows = load_labels(symbol_path) if os.path.isfile(symbol_path) else [] + whitelist_symbols = [x[0] for x in whitelist_rows if x] + self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + ( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)) + | (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)) + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/telephone.py b/nemo_text_processing/text_normalization/pt/taggers/telephone.py new file mode 100644 index 000000000..a24e30c67 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/telephone.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_SPACE, NEMO_WHITE_SPACE, GraphFst, insert_space +from nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying pt-BR telephone and IP formats, e.g. + (11) 99999-8888 -> telephone { number_part: "um um nove nove nove nove nove oito oito oito oito" } + +55 11 3333-4444 -> telephone { country_code: "mais cinco cinco" number_part: "um um três três três três quatro quatro quatro quatro" } + 192.168.1.1 -> telephone { number_part: "um nove dois ponto um seis oito ponto um ponto um" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize() + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize() + single_digits = (digit | zero).optimize() + + # Only strip grouping punctuation between digit blocks; do not delete spaces + # (deleting spaces would glue spoken digit words together in the output). + separators = pynini.union("-", ".") + # Optional separator after country / prompt (still allow +55 11 …). + delete_optional_sep = pynini.closure(pynutil.delete(separators), 0, 1) + # Require an explicit separator between local digit blocks so plain long cardinals + # (e.g. 3022110709) are not misclassified as telephone. + delete_required_separator = pynutil.delete(separators) + delete_optional_spaces = pynini.closure(pynutil.delete(NEMO_WHITE_SPACE), 0) + + def n_digits(n: int): + return pynini.closure(single_digits + insert_space, n - 1, n - 1) + single_digits + + country_digits = n_digits(1) | n_digits(2) | n_digits(3) + country_code = pynini.cross("+", "mais ") + country_digits + + ip_prompts = pynini.string_file(get_abs_path("data/telephone/ip_prompt.tsv")) + telephone_prompts = pynini.string_file(get_abs_path("data/telephone/telephone_prompt.tsv")) + tel_prompt_sequence = telephone_prompts + NEMO_SPACE + pynini.closure(country_code, 0, 1) + + country_code_graph = ( + pynutil.insert('country_code: "') + + (country_code | ip_prompts | tel_prompt_sequence) + + delete_optional_sep + + pynutil.insert('"') + ) + + area_code = (pynutil.delete("(") + n_digits(2) + pynutil.delete(")")) | n_digits(2) + + eleven_digit_graph = ( + area_code + + delete_optional_spaces + + insert_space + + n_digits(5) + + delete_required_separator + + insert_space + + n_digits(4) + ) + ten_digit_graph = ( + area_code + + delete_optional_spaces + + insert_space + + n_digits(4) + + delete_required_separator + + insert_space + + n_digits(4) + ) + nine_digit_graph = n_digits(5) + delete_required_separator + insert_space + n_digits(4) + eight_digit_graph = n_digits(4) + delete_required_separator + insert_space + n_digits(4) + seven_digit_graph = n_digits(3) + delete_required_separator + insert_space + n_digits(4) + + digit_to_str_graph = single_digits + pynini.closure(pynutil.insert(" ") + single_digits, 0, 2) + ip_graph = digit_to_str_graph + (pynini.cross(".", " ponto ") + digit_to_str_graph) ** 3 + + number_part = ( + eleven_digit_graph + | ten_digit_graph + | nine_digit_graph + | eight_digit_graph + | seven_digit_graph + | pynutil.add_weight(ip_graph, 0.01) + ) + number_part = pynutil.insert('number_part: "') + number_part + pynutil.insert('"') + + # "ramal" -> spoken "ramal …"; "extensão" / "ext." -> spoken "extensão …" (not "ext." letter-by-letter). + ext_core = n_digits(1) + pynini.closure(insert_space + n_digits(1), 0, 3) + extension_intro = delete_optional_spaces + ( + (pynutil.delete("ramal") + delete_optional_spaces + pynutil.insert("ramal ")) + | ( + (pynutil.delete("extensão") | pynutil.delete("ext.")) + + delete_optional_spaces + + pynutil.insert("extensão ") + ) + ) + ext_graph = pynutil.insert('extension: "') + extension_intro + ext_core + pynutil.insert('"') + + graph = ( + pynini.closure(country_code_graph + delete_optional_spaces + insert_space, 0, 1) + + number_part + + pynini.closure(delete_optional_spaces + insert_space + ext_graph, 0, 1) + ) + + self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/time.py b/nemo_text_processing/text_normalization/pt/taggers/time.py new file mode 100644 index 000000000..4e2f84165 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/time.py @@ -0,0 +1,180 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying Portuguese (Brazilian) time, e.g. + 14:30 -> time { hours: "catorze" minutes: "trinta" preserve_order: true } + 14:30:05 -> time { hours: "catorze" minutes: "trinta" seconds: "cinco" preserve_order: true } + 09:00:31 -> time { hours: "nove" minutes: "zero" seconds: "trinta e um" preserve_order: true } + 12:00 -> time { hours: "doze" preserve_order: true } + 11:00 da manhã -> time { hours: "onze" suffix: "da manhã" preserve_order: true } + 16:00 da tarde -> time { hours: "quatro" suffix: "da tarde" preserve_order: true } + 23:18 da tarde -> time { hours: "vinte e três" ... suffix: "da tarde" preserve_order: true } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + cardinal_graph = cardinal.graph.optimize() + + hour_words = [] + for h in range(24): + key = str(h) + comp = pynini.compose(pynini.accep(key), cardinal_graph).optimize() + hour_words.append(pynini.shortestpath(comp, nshortest=1, unique=True).string()) + + hour_delete_fsts = [] + for h in range(24): + if h < 10: + hour_delete_fsts.append(pynini.union(pynutil.delete(str(h)), pynutil.delete(f"0{h}")).optimize()) + else: + hour_delete_fsts.append(pynutil.delete(str(h))) + + labels_minute_single = [str(x) for x in range(1, 10)] + labels_minute_double = [str(x) for x in range(10, 60)] + + graph_minute_single = pynini.union(*labels_minute_single) @ cardinal_graph + graph_minute_double = pynini.union(*labels_minute_double) @ cardinal_graph + final_graph_minute = ( + pynutil.insert('minutes: "') + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert('"') + ) + + final_graph_second = ( + pynutil.insert('seconds: "') + + (pynutil.delete("0") + graph_minute_single | graph_minute_double) + + pynutil.insert('"') + ) + + # HMS verbalizer always expects ``minutes`` and ``seconds`` tags; bare ``delete("00")`` omits them. + zero_word = hour_words[0] + minutes_zero = ( + pynutil.delete("00") + pynutil.insert('minutes: "') + pynutil.insert(zero_word) + pynutil.insert('"') + ) + seconds_zero = ( + pynutil.delete("00") + pynutil.insert('seconds: "') + pynutil.insert(zero_word) + pynutil.insert('"') + ) + + delete_h = pynini.union( + pynutil.delete(pynini.accep(pynini.escape("h"))), + pynutil.delete(pynini.accep(pynini.escape("H"))), + ) + + time_delim = pynini.union( + pynini.accep(pynini.escape(":")), + pynini.accep(pynini.escape(".")), + ) + + period_rows = load_labels(get_abs_path("data/time/day_period_suffix.tsv")) + period_meta = [] + for row in period_rows: + if len(row) < 2 or not row[0].strip(): + continue + tail, tag_val = row[0].strip(), row[1].strip() + if len(row) < 4 or not row[2].strip().isdigit() or not row[3].strip().isdigit(): + raise ValueError( + f"day_period_suffix.tsv row must have 4 columns (tail, tag, hour_min, hour_max): {row!r}" + ) + h0, h1 = int(row[2].strip()), int(row[3].strip()) + allowed = frozenset(range(h0, h1 + 1)) + suf_fst = insert_space + delete_space + pynutil.delete("da") + delete_space + pynutil.delete(tail) + period_meta.append((tag_val, allowed, suf_fst, tail)) + + preserve = pynutil.insert(" preserve_order: true") + + mid_hm = pynutil.delete(time_delim) + (pynutil.delete("00") | insert_space + final_graph_minute) + mid_h_minute = delete_h + (pynutil.delete("00") | insert_space + final_graph_minute) + mid_h_only = delete_h + mid_hms = ( + pynutil.delete(time_delim) + + (minutes_zero | insert_space + final_graph_minute) + + pynutil.delete(time_delim) + + (seconds_zero | insert_space + final_graph_second) + ) + + graph_chunks = [] + for mid_after_hour in (mid_hm, mid_h_minute, mid_h_only, mid_hms): + branches = [] + for h in range(24): + hd = hour_delete_fsts[h] + hw24 = hour_words[h] + hour_tok_24 = pynutil.insert('hours: "') + pynutil.insert(hw24) + pynutil.insert('"') + branches.append(hd + hour_tok_24 + mid_after_hour + preserve) + for tag_val, allowed, suf, tail in period_meta: + keep_suffix, hour_idx = TimeFst._resolve_suffix_hour(h, tail, allowed) + hw_suf = hour_words[hour_idx] + hour_tok_suf = pynutil.insert('hours: "') + pynutil.insert(hw_suf) + pynutil.insert('"') + if keep_suffix: + branches.append( + hd + + hour_tok_suf + + mid_after_hour + + suf + + pynutil.insert(f' suffix: "{tag_val}"') + + preserve + ) + else: + # User wrote a period: always emit ``suffix:`` so TN does not drop it from speech + # (hours stay 24h when the period does not match the clock policy). + branches.append( + hd + + hour_tok_24 + + mid_after_hour + + suf + + pynutil.insert(f' suffix: "{tag_val}"') + + preserve + ) + graph_chunks.append(pynini.union(*branches).optimize()) + + final_graph = pynini.union(*graph_chunks).optimize() + self.fst = self.add_tokens(final_graph).optimize() + + @staticmethod + def _resolve_suffix_hour(h: int, period_tail: str, allowed: frozenset) -> tuple[bool, int]: + """Return (keep_suffix, hour_index) for ``hour_words[hour_index]`` when a day-period applies.""" + if period_tail == "manhã": + allowed_m = allowed | frozenset({1, 2, 3, 4, 5}) + if h not in allowed_m: + return False, h + return True, h + if period_tail == "tarde": + if h in allowed: + return True, 12 if h == 12 else h - 12 + if 1 <= h <= 5 and (h + 12) in allowed: + return True, h + return False, h + if period_tail == "noite": + if h in allowed: + return True, h - 12 + if 6 <= h <= 11 and (h + 12) in allowed: + return True, h + return False, h + if period_tail == "madrugada": + if h in allowed: + return True, h + return False, h + return False, h diff --git a/nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..61f1ee161 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/tokenize_and_classify.py @@ -0,0 +1,135 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.pt.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pt.taggers.date import DateFst +from nemo_text_processing.text_normalization.pt.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.pt.taggers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.pt.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.pt.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.pt.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.pt.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pt.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.pt.taggers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.pt.taggers.time import TimeFst +from nemo_text_processing.text_normalization.pt.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.pt.taggers.word import WordFst +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + """ + Final class that composes all Portuguese classification grammars. This class can process an entire sentence (lower cased). + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files. + whitelist: path to a file with whitelist replacements. + """ + + def __init__( + self, + input_case: str, + deterministic: bool = False, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"_{input_case}_pt_tn_{deterministic}_deterministic{whitelist_file}.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating ClassifyFst grammars. This might take some time...") + + # Initialize Portuguese taggers + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(cardinal, deterministic=deterministic) + fraction = FractionFst(cardinal, ordinal, deterministic=deterministic) + decimal = DecimalFst(cardinal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic) + date = DateFst(cardinal, deterministic=deterministic) + time = TimeFst(cardinal, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + electronic = ElectronicFst(deterministic=deterministic) + + punctuation = PunctuationFst(deterministic=deterministic) + word_graph = WordFst(punctuation=punctuation, deterministic=deterministic).fst + whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) + + classify = ( + pynutil.add_weight(whitelist.fst, 1.01) + | pynutil.add_weight(date.fst, 1.1) + | pynutil.add_weight(time.fst, 1.1) + | pynutil.add_weight(measure.fst, 1.1) + | pynutil.add_weight(fraction.fst, 1.1) + | pynutil.add_weight(decimal.fst, 1.1) + | pynutil.add_weight(ordinal.fst, 1.1) + | pynutil.add_weight(cardinal.fst, 1.1) + | pynutil.add_weight(money.fst, 1.1) + | pynutil.add_weight(telephone.fst, 1.11) + | pynutil.add_weight(electronic.fst, 1.11) + | pynutil.add_weight(word_graph, 100) + ) + + # Wrap tokens properly + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + punct_graph = ( + pynutil.insert("tokens { ") + pynutil.add_weight(punctuation.fst, weight=2.1) + pynutil.insert(" }") + ) + + # Simple graph structure + graph = token + pynini.closure( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space) + token + ) + + # Allow punctuation + graph |= punct_graph + + self.fst = delete_space + graph + delete_space + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logger.info(f"ClassifyFst grammars are saved to {far_file}.") + + +if __name__ == "__main__": + ClassifyFst(input_case="cased", deterministic=False) diff --git a/nemo_text_processing/text_normalization/pt/taggers/whitelist.py b/nemo_text_processing/text_normalization/pt/taggers/whitelist.py new file mode 100644 index 000000000..784825165 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/whitelist.py @@ -0,0 +1,111 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.utils import augment_labels_with_punct_at_end +from nemo_text_processing.text_normalization.pt.graph_utils import ( + INPUT_CASED, + INPUT_LOWER_CASED, + NEMO_SIGMA, + NEMO_UPPER, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +def _empty_fst() -> "pynini.FstLike": + """FST that accepts nothing (no whitelist rows).""" + return pynini.intersect(pynini.accep("a"), pynini.accep("b")).optimize() + + +def get_formats(input_f, input_case=INPUT_CASED, is_default=True): + """Abbreviation format variants (same idea as EN whitelist).""" + multiple_formats = load_labels(input_f) + if not multiple_formats: + return _empty_fst() + additional_options = [] + for x, y in multiple_formats: + if input_case == INPUT_LOWER_CASED: + x = x.lower() + additional_options.append((f"{x}.", y)) + additional_options.append((f"{x[0].upper() + x[1:]}", f"{y[0].upper() + y[1:]}")) + additional_options.append((f"{x[0].upper() + x[1:]}.", f"{y[0].upper() + y[1:]}")) + multiple_formats.extend(additional_options) + + if not is_default: + multiple_formats = [(x, f"|raw_start|{x}|raw_end||norm_start|{y}|norm_end|") for (x, y) in multiple_formats] + + return pynini.string_map(multiple_formats) + + +class WhiteListFst(GraphFst): + """ + Whitelist classifier for pt-BR TN. Data lives under pt/data/whitelist/ (may be empty). + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file, keep_punct_add_end: bool = False): + whitelist = load_labels(file) if os.path.isfile(file) else [] + if not whitelist: + return _empty_fst() + if input_case == INPUT_LOWER_CASED: + whitelist = [[x.lower(), y] for x, y in whitelist] + else: + whitelist = [[x, y] for x, y in whitelist] + + if keep_punct_add_end: + whitelist.extend(augment_labels_with_punct_at_end(whitelist)) + + return pynini.string_map(whitelist) + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/tts.tsv")) + + symbol_path = get_abs_path("data/whitelist/symbol.tsv") + if os.path.isfile(symbol_path) and load_labels(symbol_path): + graph |= pynini.compose( + pynini.difference(NEMO_SIGMA, pynini.accep("/")).optimize(), + _get_whitelist_graph(input_case, symbol_path), + ).optimize() + + for x in [".", ". "]: + graph |= ( + NEMO_UPPER + + pynini.closure(pynutil.delete(x) + NEMO_UPPER, 2) + + pynini.closure(pynutil.delete("."), 0, 1) + ) + + if not deterministic: + alt_path = get_abs_path("data/whitelist/alternatives.tsv") + if os.path.isfile(alt_path) and load_labels(alt_path): + graph |= _get_whitelist_graph(input_case, alt_path, keep_punct_add_end=True) + fmt_path = get_abs_path("data/whitelist/alternatives_all_format.tsv") + if os.path.isfile(fmt_path) and load_labels(fmt_path): + graph |= get_formats(fmt_path, input_case=input_case) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + self.graph = convert_space(graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/pt/taggers/word.py b/nemo_text_processing/text_normalization/pt/taggers/word.py new file mode 100644 index 000000000..01dcc389b --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/taggers/word.py @@ -0,0 +1,81 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + MIN_NEG_WEIGHT, + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NOT_SPACE, + NEMO_SIGMA, + GraphFst, + convert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying words (pt-BR pipeline). + Same structure as the English word tagger; uses PT data paths and the passed PunctuationFst. + """ + + def __init__(self, punctuation: GraphFst, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + punct = punctuation.graph + default_graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, punct.project("input")), 1) + symbols_to_exclude = (pynini.union("$", "€", "₩", "£", "¥", "#", "%") | NEMO_DIGIT).optimize() + graph = pynini.closure(pynini.difference(NEMO_NOT_SPACE, symbols_to_exclude), 1) + graph = pynutil.add_weight(graph, MIN_NEG_WEIGHT) | default_graph + + phoneme_unit = pynini.closure(NEMO_ALPHA, 1) + pynini.closure(NEMO_DIGIT) + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.accep(pynini.escape("]")) + ) + + punct_marks = pynini.union(*punctuation.punct_marks).optimize() + stress = pynini.union("ˈ", "'", "ˌ") + ipa_phoneme_unit = pynini.string_file(get_abs_path("data/whitelist/ipa_symbols.tsv")) + ipa_phonemes = ( + pynini.closure(stress, 0, 1) + + pynini.closure(ipa_phoneme_unit, 1) + + pynini.closure(stress | ipa_phoneme_unit) + ) + delim = (punct_marks | pynini.accep(" ")) ** (1, ...) + ipa_phonemes = ipa_phonemes + pynini.closure(delim + ipa_phonemes) + pynini.closure(delim, 0, 1) + ipa_phonemes = (pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]"))).optimize() + + if not deterministic: + phoneme = ( + pynini.accep(pynini.escape("[")) + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.closure(phoneme_unit + pynini.accep(" ")) + + phoneme_unit + + pynini.closure(pynini.accep(" "), 0, 1) + + pynini.accep(pynini.escape("]")) + ).optimize() + ipa_phonemes = ( + pynini.accep(pynini.escape("[")) + ipa_phonemes + pynini.accep(pynini.escape("]")) + ).optimize() + + phoneme |= ipa_phonemes + self.graph = plurals._priority_union(convert_space(phoneme.optimize()), graph, NEMO_SIGMA) + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/pt/utils.py b/nemo_text_processing/text_normalization/pt/utils.py new file mode 100644 index 000000000..da4be3f89 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/utils.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Portuguese (PT) text normalization utilities. + +Provides get_abs_path for resolving data paths and load_labels for reading TSV label files. +""" +import csv +import os + + +def get_abs_path(rel_path: str) -> str: + """ + Resolve a path relative to this module to an absolute path. + + Args: + rel_path: path relative to the PT text normalization data directory. + + Returns: + Absolute path string. + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path: str): + """ + Load a TSV file as a list of rows (list of lists). + + Args: + abs_path: absolute path to a UTF-8 TSV file. + + Returns: + List of rows, each row a list of fields (e.g. from csv.reader). + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/__init__.py b/nemo_text_processing/text_normalization/pt/verbalizers/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/pt/verbalizers/cardinal.py new file mode 100644 index 000000000..ef102a2a9 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/cardinal.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + insert_space, + shift_cardinal_gender_pt, +) + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese cardinal numbers, e.g. + cardinal { integer: "dois" } -> dois + cardinal { integer: "dois" } -> duas (feminine context via shift_cardinal_gender_pt) + cardinal { negative: "true" integer: "cinco" } -> menos cinco + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + optional_sign = pynini.closure(pynini.cross("negative: \"true\" ", "menos") + insert_space, 0, 1) + self.optional_sign = optional_sign + + integer = pynini.closure(NEMO_NOT_QUOTE, 1) + self.integer = pynutil.delete(" \"") + integer + pynutil.delete("\"") + + integer = pynutil.delete("integer:") + self.integer + + # Generate masculine form (default) + graph_masc = optional_sign + integer + + # Generate feminine form using Portuguese gender conversion + graph_fem = shift_cardinal_gender_pt(graph_masc) + + self.graph_masc = pynini.optimize(graph_masc) + self.graph_fem = pynini.optimize(graph_fem) + + # Default to masculine for standalone numbers + # Context-aware gender selection will be handled by higher-level components + graph = graph_masc + + if not deterministic: + # For alternate renderings and contractions + # Portuguese doesn't have apocope like Spanish, but may have contractions + pass + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/date.py b/nemo_text_processing/text_normalization/pt/verbalizers/date.py new file mode 100644 index 000000000..d24a37e0e --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/date.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese (Brazilian) dates, e.g. + date { day: "quinze" month: "março" year: "dois mil e vinte e quatro" preserve_order: true } + -> quinze de março de dois mil e vinte e quatro + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + vrows = load_labels(get_abs_path("data/date/verbal_phrases.tsv")) + vp = {r[0].strip(): r[1].strip() for r in vrows if len(r) >= 2 and r[0].strip()} + prep_word = vp.get("preposition", "de") + + quoted = pynini.closure(NEMO_NOT_QUOTE, 1) + + day_expr = pynutil.delete('day: "') + quoted + pynutil.delete('"') + month_expr = pynutil.delete('month: "') + quoted + pynutil.delete('"') + year_expr = pynutil.delete('year: "') + quoted + pynutil.delete('"') + + ws = delete_space + insert_space + glue = ws + pynutil.insert(prep_word) + insert_space + ws + + graph_dmy = day_expr + glue + month_expr + glue + year_expr + delete_preserve_order + self.fst = self.delete_tokens(graph_dmy).optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/decimal.py b/nemo_text_processing/text_normalization/pt/verbalizers/decimal.py new file mode 100644 index 000000000..be2bbc80e --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/decimal.py @@ -0,0 +1,67 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese decimal numbers, e.g. + decimal { integer_part: "um" fractional_part: "vinte e seis" } -> um vírgula vinte e seis + decimal { negative: "true" integer_part: "um" ... } -> menos um vírgula ... + decimal { integer_part: "um" quantity: "milhão" } -> um milhão + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + labels = load_labels(get_abs_path("data/numbers/decimal_specials.tsv")) + spec = {r[0]: r[1] for r in labels if len(r) >= 2} + sep = spec.get("separator", "vírgula") + minus = spec.get("minus", "menos") + + optional_sign = pynini.closure(pynini.cross('negative: "true" ', minus) + insert_space, 0, 1) + + integer = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + fractional = pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + quantity = ( + delete_space + + insert_space + + pynutil.delete('quantity: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + integer_quantity = integer + quantity + decimal_part = ( + integer + + delete_space + + insert_space + + pynutil.insert(sep) + + insert_space + + fractional + + pynini.closure(quantity, 0, 1) + ) + + graph = optional_sign + pynini.union(integer_quantity, decimal_part) + + self.numbers = graph.optimize() + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/electronic.py b/nemo_text_processing/text_normalization/pt/verbalizers/electronic.py new file mode 100644 index 000000000..cbe3317b5 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/electronic.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path + +digit_no_zero = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) +graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")) +server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv")) +domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv")) +spoken_unit = pynini.string_file(get_abs_path("data/electronic/electronic_spoken_unit.tsv")) + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for verbalizing electronic. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + graph_digit = digit_no_zero | zero + + def add_space_after_char(): + return pynini.closure(NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + pynutil.insert(NEMO_SPACE)) + ( + NEMO_NOT_QUOTE - pynini.accep(NEMO_SPACE) + ) + + verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", NEMO_SIGMA) + + # Prefer whole tokens (server names, TLDs, company/common words) over letter-by-letter. + user_segment = pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | server_common | spoken_unit + user_name = ( + pynutil.delete('username: "') + + (user_segment + pynini.closure(pynutil.insert(NEMO_SPACE) + user_segment)) + + pynutil.delete('"') + ) + user_name @= verbalize_characters + + convert_defaults = ( + pynutil.add_weight(NEMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common | spoken_unit + ) + domain = convert_defaults + pynini.closure(pynutil.insert(NEMO_SPACE) + convert_defaults) + domain @= verbalize_characters + domain = pynutil.delete('domain: "') + domain + pynutil.delete('"') + + protocol = ( + pynutil.delete('protocol: "') + + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", NEMO_SIGMA) + + pynutil.delete('"') + ) + + self.graph = (pynini.closure(protocol + NEMO_SPACE, 0, 1) + domain) | ( + user_name + NEMO_SPACE + pynutil.insert("arroba" + NEMO_SPACE) + domain + | (pynutil.insert("arroba" + NEMO_SPACE) + user_name) + ) + + self.fst = self.delete_tokens(self.graph + delete_preserve_order).optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/fraction.py b/nemo_text_processing/text_normalization/pt/verbalizers/fraction.py new file mode 100644 index 000000000..fc5b286db --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/fraction.py @@ -0,0 +1,109 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese fraction numbers, e.g. + fraction { numerator: "um" denominator: "meio" morphosyntactic_features: "ordinal" } -> um meio + fraction { integer_part: "dois" numerator: "três" denominator: "quarto" } -> dois e três quartos + fraction { numerator: "dois" denominator: "onze" morphosyntactic_features: "avos" } -> dois onze avos + + Denominator routing (set by the tagger, unchanged here): + + * **Ordinal** (``morphosyntactic_features: "ordinal"``): denominators 2–10, 100, 1000 — spoken as + ordinals with plural ``s`` on the denominator when the numerator is not ``um`` (``três quartos``). + * **Avos** (``… "avos"``): all other positive integer denominators — cardinal denominator + the word + ``avos`` (``três onze avos``). Optional ``sobre`` between numerator and denominator is controlled by + ``data/fractions/specials.tsv`` key ``avos_between`` (empty = single space; ``sobre`` = ``… sobre …``). + * **Mixed** numbers use ``connector`` from the same TSV (default `` e ``) after the integer part. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + labels = load_labels(get_abs_path("data/fractions/specials.tsv")) + spec = {r[0]: r[1] for r in labels if len(r) >= 2} + connector_raw = spec.get("connector", " e ").strip() + connector = insert_space + pynutil.insert(connector_raw) + insert_space + minus = spec.get("minus", "menos ").rstrip() + plural_suffix = spec.get("plural_suffix", "s") + avos_word = spec.get("avos_suffix", " avos").strip() + avos_between = spec.get("avos_between", "").strip() + numerator_one_val = spec.get("numerator_one", "um") + denominator_half_val = spec.get("denominator_half", "meio") + + optional_sign = pynini.closure(pynini.cross('negative: "true" ', minus) + insert_space, 0, 1) + + integer = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete('" ') + + numerator_one = pynutil.delete('numerator: "') + pynini.accep(numerator_one_val) + pynutil.delete('" ') + numerator_rest = ( + pynutil.delete('numerator: "') + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE), pynini.accep(numerator_one_val)) + + pynutil.delete('" ') + ) + + denom_ordinal = ( + pynutil.delete('denominator: "') + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete('" morphosyntactic_features: "ordinal"') + ) + denom_meio = ( + pynutil.delete('denominator: "') + + pynini.accep(denominator_half_val) + + pynutil.delete('" morphosyntactic_features: "ordinal"') + ) + denom_avos = ( + pynutil.delete('denominator: "') + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete('" morphosyntactic_features: "avos"') + ) + + fraction_ordinal_singular = numerator_one + insert_space + denom_ordinal + fraction_ordinal_plural = numerator_rest + insert_space + denom_ordinal + pynutil.insert(plural_suffix) + fraction_ordinal = pynini.union(fraction_ordinal_singular, fraction_ordinal_plural) + + if avos_between: + avos_mid = insert_space + pynutil.insert(avos_between) + insert_space + else: + avos_mid = insert_space + + fraction_avos = ( + pynini.union(numerator_one, numerator_rest) + + avos_mid + + denom_avos + + insert_space + + pynutil.insert(avos_word) + ) + + fraction = pynini.union(fraction_ordinal, fraction_avos) + mixed_um_meio = integer + connector + pynutil.delete('numerator: "' + numerator_one_val + '" " ') + denom_meio + optional_integer = pynini.closure(integer + connector + insert_space, 0, 1) + graph = optional_sign + pynini.union( + pynutil.add_weight(mixed_um_meio, -0.01), + optional_integer + fraction, + ) + + self.inner_graph = graph.optimize() + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/measure.py b/nemo_text_processing/text_normalization/pt/verbalizers/measure.py new file mode 100644 index 000000000..bb94ef783 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/measure.py @@ -0,0 +1,98 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_WHITE_SPACE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure (pt-BR), e.g. + measure { cardinal { integer: "duzentos" } units: "gramas" } -> duzentos gramas + measure { cardinal { integer: "um" } units: "hora" } -> uma hora + + Args: + decimal: DecimalFst verbalizer + cardinal: CardinalFst verbalizer + fraction: FractionFst verbalizer + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + hours_unit = ( + pynutil.delete('units: "') + + pynini.union(pynini.accep("hora"), pynini.accep("horas")) + + pynutil.delete('"') + ) + non_hours_unit = ( + pynutil.delete('units: "') + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("hora", "horas")) + + pynutil.delete('"') + ) + + graph_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + cardinal.graph_fem + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + hours_unit + ) + graph_cardinal |= ( + pynutil.delete("cardinal {") + + delete_space + + cardinal.graph_masc + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + non_hours_unit + ) + + graph_decimal = ( + pynutil.delete("decimal {") + + delete_space + + decimal.numbers + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + (hours_unit | non_hours_unit) + ) + + graph_fraction = ( + pynutil.delete("fraction {") + + delete_space + + fraction.inner_graph + + delete_space + + pynutil.delete("}") + + NEMO_WHITE_SPACE + + (hours_unit | non_hours_unit) + ) + + graph = graph_cardinal | graph_decimal | graph_fraction + graph += delete_preserve_order + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/money.py b/nemo_text_processing/text_normalization/pt/verbalizers/money.py new file mode 100644 index 000000000..4b247953c --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/money.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the "License". +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_preserve_order, + insert_space, +) +from nemo_text_processing.text_normalization.pt.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money (pt-BR), e.g. + money { currency_maj: "reais" integer_part: "doze" } -> doze reais + money { ... fractional_part: "cinco" currency_min: "centavos" ... } -> doze reais e cinco centavos + + Args: + decimal: DecimalFst verbalizer (for decimal amounts embedded in money) + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + scales_data = load_labels(get_abs_path("data/numbers/scales.tsv")) + currency_plural_data = load_labels(get_abs_path("data/money/currency_major_plural.tsv")) + + scale_words = [] + for row in scales_data[1:]: + if len(row) < 2: + continue + one_label = row[0].strip() + plural = row[1].strip() + if not one_label or not plural: + continue + scale_words.extend((one_label.split()[-1], plural)) + + curr_words = [row[1].strip() for row in currency_plural_data if len(row) >= 2 and row[1].strip()] + + scales = pynini.union(*[pynini.accep(w) + NEMO_SPACE for w in scale_words]).optimize() + currencies = pynini.union(*curr_words).optimize() + + maj = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + min_unit = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + fractional_part = ( + pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ) + integer_part = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + connector_minor = pynutil.insert("e") + insert_space + if not deterministic: + connector_minor |= pynutil.insert("com") + insert_space + + graph_integer = integer_part + NEMO_SPACE + maj + + graph_integer_with_minor = ( + integer_part + + NEMO_SPACE + + maj + + NEMO_SPACE + + connector_minor + + fractional_part + + NEMO_SPACE + + min_unit + + delete_preserve_order + ) + + graph_decimal = decimal.numbers + NEMO_SPACE + maj + + graph_minor = fractional_part + NEMO_SPACE + min_unit + delete_preserve_order + + graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor + graph @= pynini.cdrewrite(pynutil.insert("de") + insert_space, scales, currencies, NEMO_SIGMA) + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/pt/verbalizers/ordinal.py new file mode 100644 index 000000000..9be8876fd --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/ordinal.py @@ -0,0 +1,47 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.pt.utils import get_abs_path + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese ordinals, e.g. + ordinal { integer: "primeiro" morphosyntactic_features: "gender_masc" } -> primeiro + ordinal { integer: "primeira" morphosyntactic_features: "gender_fem" } -> primeira (feminine rewrite applied) + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + integer = pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + fem_rewrite = pynini.string_file(get_abs_path("data/ordinals/feminine.tsv")) + feminine_rewrite = pynini.cdrewrite( + fem_rewrite, + "", + pynini.union(NEMO_SPACE, pynini.accep("[EOS]")), + NEMO_SIGMA, + ) + + graph_masc = integer + pynutil.delete(' morphosyntactic_features: "gender_masc"') + graph_fem = (integer @ feminine_rewrite) + pynutil.delete(' morphosyntactic_features: "gender_fem"') + self.fst = self.delete_tokens(pynini.union(graph_masc, graph_fem)).optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/telephone.py b/nemo_text_processing/text_normalization/pt/verbalizers/telephone.py new file mode 100644 index 000000000..eae320638 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/telephone.py @@ -0,0 +1,57 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing telephone. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + optional_country_code = pynini.closure( + pynutil.delete('country_code: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + + delete_space + + insert_space, + 0, + 1, + ) + + number_part = ( + pynutil.delete('number_part: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynini.closure(pynutil.add_weight(pynutil.delete(" "), -0.0001), 0, 1) + + pynutil.delete('"') + ) + + optional_extension = pynini.closure( + delete_space + + insert_space + + pynutil.delete('extension: "') + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"'), + 0, + 1, + ) + + graph = optional_country_code + number_part + optional_extension + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/time.py b/nemo_text_processing/text_normalization/pt/verbalizers/time.py new file mode 100644 index 000000000..dfebdab3e --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/time.py @@ -0,0 +1,125 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing Portuguese time, e.g. + time { hours: "catorze" minutes: "trinta" preserve_order: true } -> catorze horas e trinta + time { hours: "um" minutes: "trinta" preserve_order: true } -> uma hora e trinta + time { hours: "dois" minutes: "quinze" preserve_order: true } -> duas horas e quinze + time { hours: "onze" suffix: "da manhã" preserve_order: true } -> onze horas da manhã + time { hours: "vinte e um" minutes: "dezoito" suffix: "da tarde" preserve_order: true } + -> vinte e uma horas e dezoito da tarde + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + quoted = pynini.closure(NEMO_NOT_QUOTE, 1) + + minutes_val = pynutil.delete('minutes: "') + quoted + pynutil.delete('"') + seconds_val = pynutil.delete('seconds: "') + quoted + pynutil.delete('"') + suffix_val = pynutil.delete('suffix: "') + quoted + pynutil.delete('"') + + gap = delete_space + insert_space + suffix_out = pynini.closure(gap + suffix_val, 0, 1) + + hours_default = pynutil.delete('hours: "') + quoted + pynutil.delete('"') + gap + pynutil.insert("horas") + # Match whitespace after the closing quote (same as hours_default's gap) so the path composes + # with minutes/suffix fields; otherwise only the generic "… horas" branch accepts the token. + hours_um = ( + pynutil.delete('hours: "') + + pynutil.delete("um") + + pynutil.delete('"') + + delete_space + + pynutil.insert("uma hora") + ) + hours_dois = ( + pynutil.delete('hours: "') + + pynutil.delete("dois") + + pynutil.delete('"') + + delete_space + + pynutil.insert("duas horas") + ) + hours_vinte_um = ( + pynutil.delete('hours: "') + + pynutil.delete("vinte e um") + + pynutil.delete('"') + + delete_space + + pynutil.insert("vinte e uma horas") + ) + hours_vinte_dois = ( + pynutil.delete('hours: "') + + pynutil.delete("vinte e dois") + + pynutil.delete('"') + + delete_space + + pynutil.insert("vinte e duas horas") + ) + # Prefer feminine hour phrases over the generic ``… horas`` path (tie-break by weight). + hour_phrase = ( + pynutil.add_weight(hours_um, -0.01) + | pynutil.add_weight(hours_dois, -0.01) + | pynutil.add_weight(hours_vinte_um, -0.01) + | pynutil.add_weight(hours_vinte_dois, -0.01) + | hours_default + ).optimize() + + graph_hms = ( + hour_phrase + + insert_space + + pynutil.insert("e") + + insert_space + + minutes_val + + gap + + pynutil.insert("minutos") + + insert_space + + pynutil.insert("e") + + insert_space + + seconds_val + + gap + + pynutil.insert("segundos") + + suffix_out + + delete_preserve_order + ) + + with_minutes = ( + hour_phrase + + insert_space + + pynutil.insert("e") + + insert_space + + minutes_val + + suffix_out + + delete_preserve_order + ) + + hours_only = hour_phrase + suffix_out + delete_preserve_order + + graph = pynini.union(graph_hms, with_minutes, hours_only).optimize() + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/pt/verbalizers/verbalize.py new file mode 100644 index 000000000..9a6910d6a --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/verbalize.py @@ -0,0 +1,66 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.text_normalization.pt.graph_utils import GraphFst +from nemo_text_processing.text_normalization.pt.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.pt.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.pt.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.pt.verbalizers.electronic import ElectronicFst +from nemo_text_processing.text_normalization.pt.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.pt.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.pt.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.pt.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.pt.verbalizers.telephone import TelephoneFst +from nemo_text_processing.text_normalization.pt.verbalizers.time import TimeFst + + +class VerbalizeFst(GraphFst): + """ + Composes Portuguese verbalizer grammars (cardinal, ordinal, fraction, decimal). + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) + fraction = FractionFst(deterministic=deterministic) + decimal = DecimalFst(deterministic=deterministic) + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(decimal=decimal, deterministic=deterministic) + date = DateFst(deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) + electronic = ElectronicFst(deterministic=deterministic) + graph = ( + fraction.fst + | decimal.fst + | date.fst + | time.fst + | measure.fst + | money.fst + | ordinal.fst + | cardinal.fst + | telephone.fst + | electronic.fst + ) + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/pt/verbalizers/verbalize_final.py new file mode 100644 index 000000000..84ed9ac39 --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/verbalize_final.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.pt.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.pt.verbalizers.word import WordFst +from nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire Portuguese sentence, e.g. + tokens { cardinal { integer: "dois" } } tokens { name: "e" } tokens { cardinal { integer: "três" } } -> dois e três + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"pt_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/text_normalization/pt/verbalizers/word.py b/nemo_text_processing/text_normalization/pt/verbalizers/word.py new file mode 100644 index 000000000..2b53ad1ff --- /dev/null +++ b/nemo_text_processing/text_normalization/pt/verbalizers/word.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.pt.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Verbalizes tokens { name: "..." } for the pt-BR pipeline. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ru/alphabet.py b/nemo_text_processing/text_normalization/ru/alphabet.py index 3df59f468..b18f494dd 100644 --- a/nemo_text_processing/text_normalization/ru/alphabet.py +++ b/nemo_text_processing/text_normalization/ru/alphabet.py @@ -17,6 +17,7 @@ # Russian minimally supervised number grammar. import pynini + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NON_BREAKING_SPACE, NEMO_SPACE from nemo_text_processing.text_normalization.ru.utils import get_abs_path diff --git a/nemo_text_processing/text_normalization/ru/taggers/cardinal.py b/nemo_text_processing/text_normalization/ru/taggers/cardinal.py index a36500560..5e780969a 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/ru/taggers/cardinal.py @@ -17,6 +17,8 @@ # Russian minimally supervised number grammar. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -26,12 +28,11 @@ ) from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA, TO_CYRILLIC from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. "1 001" -> cardinal { integer: "тысяча один" } Args: diff --git a/nemo_text_processing/text_normalization/ru/taggers/date.py b/nemo_text_processing/text_normalization/ru/taggers/date.py index d4ce6a646..3ad16f999 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/date.py +++ b/nemo_text_processing/text_normalization/ru/taggers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_NOT_QUOTE, @@ -23,12 +25,11 @@ insert_space, ) from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class DateFst(GraphFst): """ - Finite state transducer for classifying date, e.g. + Finite state transducer for classifying date, e.g. "01.05" -> tokens { date { day: "первое мая" } } Args: @@ -77,7 +78,7 @@ def __init__(self, number_names: dict, deterministic: bool): month = ( pynutil.insert("month: \"") + (month_name | pynutil.add_weight(digit_month, 0.1)) + pynutil.insert("\"") ).optimize() - year = pynini.compose(((NEMO_DIGIT ** 4) | (NEMO_DIGIT ** 2)), numbers).optimize() + year = pynini.compose(((NEMO_DIGIT**4) | (NEMO_DIGIT**2)), numbers).optimize() year |= zero_digit # reduce year options diff --git a/nemo_text_processing/text_normalization/ru/taggers/decimals.py b/nemo_text_processing/text_normalization/ru/taggers/decimals.py index de29513a3..40ced8d52 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/decimals.py +++ b/nemo_text_processing/text_normalization/ru/taggers/decimals.py @@ -15,10 +15,11 @@ from collections import defaultdict import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, insert_space from nemo_text_processing.text_normalization.en.utils import load_labels from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil delete_space = pynutil.delete(" ") @@ -49,7 +50,7 @@ def prepare_labels_for_insertion(file_path: str): class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. + Finite state transducer for classifying decimal, e.g. "1,08" -> tokens { decimal { integer_part: "одно целая" fractional_part: "восемь сотых} } Args: diff --git a/nemo_text_processing/text_normalization/ru/taggers/electronic.py b/nemo_text_processing/text_normalization/ru/taggers/electronic.py index 22530a28b..f0a5e7bcd 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/ru/taggers/electronic.py @@ -14,6 +14,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA, TO_CYRILLIC from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class ElectronicFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/taggers/measure.py b/nemo_text_processing/text_normalization/ru/taggers/measure.py index a4c60f49c..86c92748f 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/measure.py +++ b/nemo_text_processing/text_normalization/ru/taggers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NON_BREAKING_SPACE, NEMO_NOT_QUOTE, @@ -22,7 +24,6 @@ ) from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class MeasureFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/taggers/money.py b/nemo_text_processing/text_normalization/ru/taggers/money.py index c08723ae8..4adcd76fd 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/money.py +++ b/nemo_text_processing/text_normalization/ru/taggers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, NEMO_SPACE, GraphFst from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/taggers/number_names.py b/nemo_text_processing/text_normalization/ru/taggers/number_names.py index 048b5f43e..4ee630448 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/number_names.py +++ b/nemo_text_processing/text_normalization/ru/taggers/number_names.py @@ -26,9 +26,10 @@ # acceptor (G). import pynini +from pynini.lib import pynutil, rewrite + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA from nemo_text_processing.text_normalization.ru.utils import get_abs_path, load_labels -from pynini.lib import pynutil, rewrite def get_number_names(): diff --git a/nemo_text_processing/text_normalization/ru/taggers/ordinal.py b/nemo_text_processing/text_normalization/ru/taggers/ordinal.py index 9a09e4063..43277db46 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/ru/taggers/ordinal.py @@ -17,14 +17,15 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. + Finite state transducer for classifying cardinals, e.g. "2" -> ordinal { integer: "второе" } } Args: diff --git a/nemo_text_processing/text_normalization/ru/taggers/telephone.py b/nemo_text_processing/text_normalization/ru/taggers/telephone.py index 142242e8a..456bd6f1a 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/ru/taggers/telephone.py @@ -13,16 +13,17 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_DIGIT, GraphFst, delete_space, insert_space from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA_OR_SPACE -from pynini.lib import pynutil class TelephoneFst(GraphFst): """ - Finite state transducer for classifying telephone, which includes country code, number part and extension + Finite state transducer for classifying telephone, which includes country code, number part and extension - E.g + E.g "8-913-983-56-01" -> telephone { number_part: "восемь девятьсот тринадцать девятьсот восемьдесят три пятьдесят шесть ноль один" } Args: @@ -47,13 +48,13 @@ def __init__(self, number_names: dict, deterministic: bool = True): optional_country_code = pynini.closure(country_code + insert_space, 0, 1) number_part = ( - NEMO_DIGIT ** 3 @ number + NEMO_DIGIT**3 @ number + separator - + NEMO_DIGIT ** 3 @ number + + NEMO_DIGIT**3 @ number + separator - + NEMO_DIGIT ** 2 @ number + + NEMO_DIGIT**2 @ number + separator - + NEMO_DIGIT ** 2 @ (pynini.closure(pynini.cross("0", "ноль ")) + number) + + NEMO_DIGIT**2 @ (pynini.closure(pynini.cross("0", "ноль ")) + number) ) number_part = pynutil.insert("number_part: \"") + number_part + pynutil.insert("\"") tagger_graph = (optional_country_code + number_part).optimize() diff --git a/nemo_text_processing/text_normalization/ru/taggers/time.py b/nemo_text_processing/text_normalization/ru/taggers/time.py index bb97f36ea..427aab00d 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/time.py +++ b/nemo_text_processing/text_normalization/ru/taggers/time.py @@ -14,16 +14,17 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.ru.utils import get_abs_path -from pynini.lib import pynutil class TimeFst(GraphFst): """ Finite state transducer for classifying time, e.g. "02:15" -> time { hours: "два часа пятнадцать минут" } - + Args: number_names: number_names for cardinal and ordinal numbers deterministic: if True will provide a single transduction option, diff --git a/nemo_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py index bd7bb4e61..53f2b41f5 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ru/taggers/tokenize_and_classify.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -35,7 +36,7 @@ from nemo_text_processing.text_normalization.ru.taggers.time import TimeFst from nemo_text_processing.text_normalization.ru.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.ru.taggers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -75,9 +76,9 @@ def __init__( ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f"ClassifyFst.fst was restored from {far_file}.") + logger.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") + logger.info(f"Creating ClassifyFst grammars. This might take some time...") number_names = get_number_names() alternative_formats = get_alternative_formats() @@ -138,4 +139,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ru/taggers/whitelist.py b/nemo_text_processing/text_normalization/ru/taggers/whitelist.py index 851829208..e4ca3a0c5 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/ru/taggers/whitelist.py @@ -13,10 +13,11 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_CHAR, GraphFst, convert_space from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA, TO_CYRILLIC from nemo_text_processing.text_normalization.ru.utils import get_abs_path, load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/taggers/word.py b/nemo_text_processing/text_normalization/ru/taggers/word.py index 16743d28b..7e0df2a95 100644 --- a/nemo_text_processing/text_normalization/ru/taggers/word.py +++ b/nemo_text_processing/text_normalization/ru/taggers/word.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ru/utils.py b/nemo_text_processing/text_normalization/ru/utils.py index 7eda39c38..a55659868 100644 --- a/nemo_text_processing/text_normalization/ru/utils.py +++ b/nemo_text_processing/text_normalization/ru/utils.py @@ -13,9 +13,10 @@ # limitations under the License. import csv -import logging import os +from nemo_text_processing.utils.logging import logger + def get_abs_path(rel_path): """ @@ -23,13 +24,13 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path if not os.path.exists(abs_path): - logging.warning(f'{abs_path} does not exist') + logger.warning(f'{abs_path} does not exist') return abs_path @@ -42,6 +43,6 @@ def load_labels(abs_path): Returns dictionary of mappings """ - label_tsv = open(abs_path, encoding='utf-8') - labels = list(csv.reader(label_tsv, delimiter="\t")) + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) return labels diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/ru/verbalizers/cardinal.py index 4bff9ed3f..7be8d9727 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/date.py b/nemo_text_processing/text_normalization/ru/verbalizers/date.py index 52a69b0c9..82ba1b35f 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/date.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/decimal.py b/nemo_text_processing/text_normalization/ru/verbalizers/decimal.py index c3006fe3d..a2900bd57 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DecimalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/electronic.py b/nemo_text_processing/text_normalization/ru/verbalizers/electronic.py index 5bbfe008d..8440150c5 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA -from pynini.lib import pynutil class ElectronicFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/measure.py b/nemo_text_processing/text_normalization/ru/verbalizers/measure.py index 27c95b9ee..001691518 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/measure.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NON_BREAKING_SPACE, NEMO_SPACE, @@ -20,14 +22,13 @@ delete_space, ) from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA -from pynini.lib import pynutil class MeasureFst(GraphFst): """ Finite state transducer for verbalizing measure, e.g. measure { cardinal { integer: "два килограма" } } -> "два килограма" - + Args: deterministic: if True will provide a single transduction option, for False multiple transduction are generated (used for audio-based normalization) diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/money.py b/nemo_text_processing/text_normalization/ru/verbalizers/money.py index 3f5a5e936..02b903407 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/money.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/ru/verbalizers/ordinal.py index 00700d9ec..8f4288465 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/telephone.py b/nemo_text_processing/text_normalization/ru/verbalizers/telephone.py index bbd2d29f0..fac111ec8 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/telephone.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/telephone.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst from nemo_text_processing.text_normalization.ru.alphabet import RU_ALPHA -from pynini.lib import pynutil class TelephoneFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/time.py b/nemo_text_processing/text_normalization/ru/verbalizers/time.py index 4f72879d2..9cbdb0e21 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/time.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + class TimeFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/ru/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ru/verbalizers/verbalize_final.py index 9bb446448..ceaf04d7d 100644 --- a/nemo_text_processing/text_normalization/ru/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/ru/verbalizers/verbalize_final.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -23,12 +24,12 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.ru.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): """ - Finite state transducer that verbalizes an entire sentence, e.g. + Finite state transducer that verbalizes an entire sentence, e.g. tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now Args: @@ -47,7 +48,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"ru_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst().fst @@ -67,4 +68,3 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/run_evaluate.py b/nemo_text_processing/text_normalization/run_evaluate.py index 8204eccbf..3a1964bbd 100644 --- a/nemo_text_processing/text_normalization/run_evaluate.py +++ b/nemo_text_processing/text_normalization/run_evaluate.py @@ -23,7 +23,6 @@ ) from nemo_text_processing.text_normalization.normalize import Normalizer - ''' Runs Evaluation on data in the format of : \t\t<`self` if trivial class or normalized text> like the Google text normalization data https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish @@ -36,7 +35,7 @@ def parse_args(): parser.add_argument( "--lang", help="language", - choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh'], + choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'], default="en", type=str, ) @@ -65,7 +64,8 @@ def parse_args(): normalizer = Normalizer(input_case=args.input_case, lang=args.lang) print("Loading training data: " + file_path) - training_data = load_files([file_path]) + to_lower = args.input_case == "lower_cased" + training_data = load_files([file_path], to_lower=to_lower) if args.filter: training_data = filter_loaded_data(training_data) @@ -75,6 +75,9 @@ def parse_args(): sentences_un_normalized, sentences_normalized, _ = training_data_to_sentences(training_data) print("- Data: " + str(len(sentences_normalized)) + " sentences") sentences_prediction = normalizer.normalize_list(sentences_un_normalized) + with open('result.log', 'w') as ofp: + for inp, out in zip(sentences_normalized, sentences_prediction): + ofp.write(f'{inp==out}; {inp}\t{out}\n') print("- Normalized. Evaluating...") sentences_accuracy = evaluate( preds=sentences_prediction, labels=sentences_normalized, input=sentences_un_normalized @@ -101,8 +104,6 @@ def parse_args(): print("- Accuracy: " + str(sum(token_weighted_accuracy) / sum(token_count_per_type.values()))) print(" - Total: " + str(sum(token_count_per_type.values())), '\n') - print(" - Total: " + str(sum(token_count_per_type.values())), '\n') - for token_type in token_accuracy: if token_type not in known_types: raise ValueError("Unexpected token type: " + token_type) diff --git a/nemo_text_processing/text_normalization/rw/__init__.py b/nemo_text_processing/text_normalization/rw/__init__.py new file mode 100644 index 000000000..876f20b3f --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nemo_text_processing.text_normalization.rw.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.text_normalization.rw.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.rw.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/text_normalization/rw/data/__init__.py b/nemo_text_processing/text_normalization/rw/data/__init__.py new file mode 100644 index 000000000..9c4313114 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/__init__.py b/nemo_text_processing/text_normalization/rw/data/cardinal/__init__.py new file mode 100644 index 000000000..9c4313114 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/digits.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/digits.tsv new file mode 100644 index 000000000..bf85b743b --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/digits.tsv @@ -0,0 +1,9 @@ +rimwe 1 +kabiri 2 +gatatu 3 +kane 4 +gatanu 5 +gatandatu 6 +karindwi 7 +umunani 8 +icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/digits_for_thousands.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/digits_for_thousands.tsv new file mode 100644 index 000000000..ee31aadee --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/digits_for_thousands.tsv @@ -0,0 +1,10 @@ + 0 +kimwe 1 +bibiri 2 +bitatu 3 +bine 4 +bitanu 5 +bitandatu 6 +birindwi 7 +umunani 8 +icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/digits_millions_trillions.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/digits_millions_trillions.tsv new file mode 100644 index 000000000..126ad90a3 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/digits_millions_trillions.tsv @@ -0,0 +1,10 @@ + 0 +imwe 1 +ebyiri 2 +eshatu 3 +enye 4 +eshanu 5 +esheshatu 6 +zirindwi 7 +umunani 8 +icyenda 9 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds.tsv new file mode 100644 index 000000000..a46623cc1 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds.tsv @@ -0,0 +1,9 @@ +ijana 1 +magana_abiri 2 +magana_atatu 3 +magana_ane 4 +magana_atanu 5 +magana_atandatu 6 +magana_arindwi 7 +magana_inani 8 +magana_cyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_millions.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_millions.tsv new file mode 100644 index 000000000..6e38c3ceb --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_millions.tsv @@ -0,0 +1,9 @@ +miliyoni_ijana 1 +miliyoni_magana_abiri 2 +miliyoni_magana_atatu 3 +miliyoni_magana_ane 4 +miliyoni_magana_atanu 5 +miliyoni_magana_atandatu 6 +miliyoni_magana_arindwi 7 +miliyoni_magana_inani 8 +miliyoni_magana_cyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_thousands.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_thousands.tsv new file mode 100644 index 000000000..a73477c14 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_thousands.tsv @@ -0,0 +1,9 @@ +ibihumbi_ijana 1 +ibihumbi_magana_abiri 2 +ibihumbi_magana_atatu 3 +ibihumbi_magana_ane 4 +ibihumbi_magana_atanu 5 +ibihumbi_magana_atandatu 6 +ibihumbi_magana_arindwi 7 +ibihumbi_magana_inani 8 +ibihumbi_magana_cyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_trillions.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_trillions.tsv new file mode 100644 index 000000000..00fc01aa4 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/hundreds_of_trillions.tsv @@ -0,0 +1,9 @@ +tiriyoni_ijana 1 +tiriyoni_magana_abiri 2 +tiriyoni_magana_atatu 3 +tiriyoni_magana_ane 4 +tiriyoni_magana_atanu 5 +tiriyoni_magana_atandatu 6 +tiriyoni_magana_arindwi 7 +tiriyoni_magana_inani 8 +tiriyoni_magana_cyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/millions.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/millions.tsv new file mode 100644 index 000000000..fded5ed55 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/millions.tsv @@ -0,0 +1,9 @@ +miliyoni 1 +miliyoni_ebyiri 2 +miliyoni_eshatu 3 +miliyoni_enye 4 +miliyoni_eshanu 5 +miliyoni_esheshatu 6 +miliyoni_zirindwi 7 +miliyoni_umunani 8 +miliyoni_icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/tens.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/tens.tsv new file mode 100644 index 000000000..6e63c3875 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/tens.tsv @@ -0,0 +1,9 @@ + 0 +makumyabiri 2 +mirongo_itatu 3 +mirongo_ine 4 +mirongo_itanu 5 +mirongo_itandatu 6 +mirongo_irindwi 7 +mirongo_inani 8 +mirongo_icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_millions.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_millions.tsv new file mode 100644 index 000000000..36f077d00 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_millions.tsv @@ -0,0 +1,9 @@ +miliyoni_cumi 1 +miliyoni_makumyabiri 2 +miliyoni_mirongo_itatu 3 +miliyoni_mirongo_ine 4 +miliyoni_mirongo_itanu 5 +miliyoni_mirongo_itandatatu 6 +miliyoni_mirongo_irindwi 7 +miliyoni_mirongo_inani 8 +miliyoni_mirongo_icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_thousands.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_thousands.tsv new file mode 100644 index 000000000..f230751bf --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_thousands.tsv @@ -0,0 +1,9 @@ +ibihumbi_cumi 1 +ibihumbi_makumyabiri 2 +ibihumbi_mirongo_itatu 3 +ibihumbi_mirongo_ine 4 +ibihumbi_mirongo_itanu 5 +ibihumbi_mirongo_itandatatu 6 +ibihumbi_mirongo_irindwi 7 +ibihumbi_mirongo_inani 8 +ibihumbi_mirongo_icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_trillions.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_trillions.tsv new file mode 100644 index 000000000..3cf483594 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/tens_of_trillions.tsv @@ -0,0 +1,9 @@ +tiriyoni_icumi 1 +tiriyoni_makumyabiri 2 +tiriyoni_mirongo_itatu 3 +tiriyoni_mirongo_ine 4 +tiriyoni_mirongo_itanu 5 +tiriyoni_mirongo_itandatatu 6 +tiriyoni_mirongo_irindwi 7 +tiriyoni_mirongo_inani 8 +tiriyoni_mirongo_icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/thousands.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/thousands.tsv new file mode 100644 index 000000000..39d262443 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/thousands.tsv @@ -0,0 +1,10 @@ +igihumbi 1 +ibihumbi_bibiri 2 +ibihumbi_bitatu 3 +ibihumbi_bine 4 +ibihumbi_bitanu 5 +ibihumbi_bitandatu 6 +ibihumbi_birindwi 7 +ibihumbi_umunani 8 +ibihumbi_icyenda 9 + diff --git a/nemo_text_processing/text_normalization/rw/data/cardinal/trillions.tsv b/nemo_text_processing/text_normalization/rw/data/cardinal/trillions.tsv new file mode 100644 index 000000000..8098158df --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/cardinal/trillions.tsv @@ -0,0 +1,9 @@ +tiriyoni 1 +tiriyoni_ebyiri 2 +tiriyoni_eshatu 3 +tiriyoni_enye 4 +tiriyoni_eshanu 5 +tiriyoni_esheshatu 6 +tiriyoni_zirindwi 7 +tiriyoni_umunani 8 +tiriyoni_icyenda 9 diff --git a/nemo_text_processing/text_normalization/rw/data/time/__init__.py b/nemo_text_processing/text_normalization/rw/data/time/__init__.py new file mode 100644 index 000000000..9c4313114 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/time/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/rw/data/time/hours.tsv b/nemo_text_processing/text_normalization/rw/data/time/hours.tsv new file mode 100644 index 000000000..fae6f0898 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/time/hours.tsv @@ -0,0 +1,12 @@ +1 saa saba +2 saa munani +3 saa cyenda +4 saa cumi +5 saa cumi n'imwe +6 saa cumi n'ebyiri +7 saa moya +8 saa mbiri +9 saa tatu +10 saa ine +11 saa tanu +12 saa sita \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/rw/data/time/minutes.tsv b/nemo_text_processing/text_normalization/rw/data/time/minutes.tsv new file mode 100644 index 000000000..c30327106 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/time/minutes.tsv @@ -0,0 +1,60 @@ +00 +01 n'umunota umwe +02 n'iminota ibiri +03 n'iminota itatu +04 n'iminota ine +05 n'iminota itanu +06 n'iminota itandatu +07 n'iminota irindwi +08 n'iminota umunani +09 n'iminota icyenda +10 n'iminota icumi +11 n'iminota cumi n'umwe +12 n'iminota cumi n'ibiri +13 n'iminota cumi n'itatu +14 n'iminota cumi n'ine +15 n'iminota cumi n'itanu +16 n'iminota cumi n'itandatu +17 n'iminota cumi n'irindwi +18 n'iminota cumi n'umunani +19 n'iminota cumi n'icyenda +20 n'iminota makumyabiri +21 n'iminota makumyabiri na rimwe +22 n'iminota makumyabiri n'ibiri +23 n'iminota makumyabiri n'itatu +24 n'iminota makumyabiri n'ine +25 n'iminota makumyabiri n'itanu +26 n'iminota makumyabiri n'itandatu +27 n'iminota makumyabiri n'irindwi +28 n'iminota makumyabiri n'umunani +29 n'iminota makumyabiri n'icyenda +30 n'iminota mirongo itatu +31 n'iminota mirongo itatu n'umwe +32 n'iminota mirongo itatu n'ibiri +33 n'iminota mirongo itatu n'itatu +34 n'iminota mirongo itatu n'ine +35 n'iminota mirongo itatu n'itanu +36 n'iminota mirongo itatu n'itandatu +37 n'iminota mirongo itatu n'irindwi +38 n'iminota mirongo itatu n'umunani +39 n'iminota mirongo itatu n'icyenda +40 n'iminota mirongo ine +41 n'iminota mirongo ine n'umwe +42 n'iminota mirongo ine n'ibiri +43 n'iminota mirongo ine n'itatu +44 n'iminota mirongo ine n'ine +45 n'iminota mirongo ine n'itanu +46 n'iminota mirongo ine n'itandatu +47 n'iminota mirongo ine n'irindwi +48 n'iminota mirongo ine n'umunani +49 n'iminota mirongo ine n'icyenda +50 n'iminota mirongo itanu +51 n'iminota mirongo itanu n'umwe +52 n'iminota mirongo itanu n'ibiri +53 n'iminota mirongo itanu n'itatu +54 n'iminota mirongo itanu n'ine +55 n'iminota mirongo itanu n'itanu +56 n'iminota mirongo itanu n'itandatu +57 n'iminota mirongo itanu n'irindwi +58 n'iminota mirongo itanu n'umunani +59 n'iminota mirongo itanu n'icyenda \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/rw/data/whitelist/__init__.py b/nemo_text_processing/text_normalization/rw/data/whitelist/__init__.py new file mode 100644 index 000000000..9c4313114 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/whitelist/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/rw/data/whitelist/kinya_transliterations.tsv b/nemo_text_processing/text_normalization/rw/data/whitelist/kinya_transliterations.tsv new file mode 100644 index 000000000..e550214cd --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/data/whitelist/kinya_transliterations.tsv @@ -0,0 +1,175 @@ +Chris kiris +Hipkins Hipikineze +Dexamethasone dekisametazone +corticosteroid koritikositeroyide +immunosuppressant iminosupuresa +CDC sidisi +RBC arabisi +RISA risa +minisante minisante +sars sarisi +pfizer pifiza +BionTech biyoniteki +dollar dorari +ADN ade eni +BBC bibisi +Victoria vikitoria +Espagne esipanye +Nouvelle-Zélande nuveli zerandi +lopinavir lopinaviri +rotinavir rotinaviri +HIV heci ayivi +seychelles seyisheli +maroc maroki +sputnik siputinike +la crosse la korosi +paul pawulo +www wawawa +gov govu +rw rwa +http hecititipi +Berlin iberile +Remdesivir remudesivire +coranavirus koronavirusi +covid kovide +quarantine karantine +oms o e mesi +basketball basiketibalu +football futibolu +cholera kolera +radio radiyo +television televiziyo +service serivise +prof purofu +royal ruyolo +college koreji +health ubuzima +SARS-CoV-kabiri sarisi-kov-kabiri +recovery rekoveri +Dr dogiteri +protein puroteyine +spike sipiyike +victoria vigitoriya +technique tekinike +cell selile +electro erekitoro +sanitizer sanitayiza +Orthocoronavirinae oritocoronavirinaye +coronavirinae coronavirinaye +nidovirales nidoviralesi +Covs covuse +antibody antibodi +Hydroxychloroquine hidurokulorokine +company kompani +oxygen ogisijeni +Carolina karolina +jonathan jonatani +hyper hiperi +micro mikoro +microscope mikorosikope +microchip mikorocipu +glycoproteine gilicopuroteyine +sport siporo +lockdown lokidawuno +email imeli +japan japani +science siyansi +pubmed pubimedi +koica koyika +jica jika +DNA diyeniyi +RNA araneyi +wuhan wuhani +huanan hwanani +thermoregulation terimoregulashiyoni +alveolar aliviyola +hypoxemia hipokisemiya +PCR pisiyara +rapid-test rapidi-tesite +sepsis sepusisi +septique seputike +pulmonary pirimonari +extra egisitura +Real riyo +Time tayimu +Polymerase porimerase +poly pori +Chain ceyini +Reaction reyakishoni +hypoxic hipokisike +ICU ayisiyu +ambulance amburansi +antibiotic antibiyotike +vaccine vagisine +MEDAIR medayire +guardian garidiyani +covax covagise +paris parisi +transplant turansipulanti +laboratoire laboratuwari +Tedros tewodurosi +Ghebreyesus gebureyesusi +polybasic poribazike +china chinwa +RT-PCR arati-pisiyara +UNICEF yunicefu +HCR hashiseyeri +UNESCO yunesico +UN oni +World woridi +bank banki +FMI efu emi +new-york nuyoriki +times tayimuze +MERS merise +electron erekituronu +RDB aradibi +Platelet-fibrin puratele-fibirini +arterial ariteriyo +coagulopathie kowagulopati +RBD arabidi +RDF aradiyefu +module modile +Oxford ogisiforudu +AstraZeneca asutarazeneka +Astra-Zeneca asutarazeneka +astra asutara +zeneca zeneka +chlorine kulorakine +acide aside +peroxyacetic perukisiyatike +chlorhexidine kulorekidine +chloroform kuloroforume +disinfectant dezenkifekita +carbon kariboni +Hopkins hopikinze +communist komunisite +Tanzania tanzaniya +Africa afurika +VOA vi o aye +Jean yohana +Marie mariya +Vianney viyane +chimiotherapie kimyoterapi +sinopharm sinofarume +bus busi +ventilator ventirata +ventilators ventirataze +mRNA emu araneyi +Favipiravir favipiravire +command komandi +center santire +app apu +phone fone +telephone terefone +clinical kiliniko +clinique kilinike +lymphocytes lemfosite +twitter tuwita +youtube yutubi +facebook fasibuki +google gugoli +com komu +Antibodies antibodize +COVID-CUMI kovide-cumi +COVID-CUMI-N'ICYENDA kovide-cumi-n'icyenda diff --git a/nemo_text_processing/text_normalization/rw/graph_utils.py b/nemo_text_processing/text_normalization/rw/graph_utils.py new file mode 100644 index 000000000..ecb20fe84 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/graph_utils.py @@ -0,0 +1,297 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_VOWELS = pynini.union(*"aeiouAEIOU").optimize() +NEMO_CONSONANTS = pynini.union(*"BCDFGHJKLMNPQRSTVWXYZbcdfghjklmnpqrstvwxyz").optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_space_or_punct = NEMO_PUNCT | delete_space +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + + +# Common string literals; expand as you see fit. +username_string = "username" +double_quotes = '"' +domain_string = "domain" +protocol_string = "protocol" +slash = "/" +double_slash = "//" +triple_slash = "///" +file = "file" +period = "." +at = "@" +colon = ":" +https = "https" +http = "http" +www = "www" + + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/rw/taggers/__init__.py b/nemo_text_processing/text_normalization/rw/taggers/__init__.py new file mode 100644 index 000000000..9c4313114 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/taggers/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/rw/taggers/cardinal.py b/nemo_text_processing/text_normalization/rw/taggers/cardinal.py new file mode 100644 index 000000000..958a95234 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/taggers/cardinal.py @@ -0,0 +1,243 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.rw.graph_utils import ( + NEMO_ALPHA, + NEMO_CHAR, + NEMO_CONSONANTS, + NEMO_DIGIT, + NEMO_VOWELS, + GraphFst, + delete_extra_space, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.rw.utils import get_abs_path + + +class CardinalFst(GraphFst): + def __init__(self): + super().__init__(name="cardinal", kind="classify") + vowels_or_space = NEMO_VOWELS | " " + rewrite_na_fst = pynini.cdrewrite( + pynini.cross(" ", " na "), vowels_or_space, NEMO_CONSONANTS, NEMO_CHAR.closure() + ) + rewrite_n_fst = pynini.cdrewrite(pynini.cross(" ", " n'"), vowels_or_space, NEMO_VOWELS, NEMO_CHAR.closure()) + remove_underscore_fst = pynini.cdrewrite( + pynini.cross("_", " "), pynini.union(NEMO_ALPHA), pynini.union(NEMO_ALPHA), NEMO_CHAR.closure() + ) + remove_extra_space_fst = pynini.cdrewrite( + delete_extra_space, pynini.union(NEMO_ALPHA), pynini.union(NEMO_ALPHA), NEMO_CHAR.closure() + ) + remove_trailing_space_fst = pynini.cdrewrite( + delete_space, pynini.union(NEMO_ALPHA).closure(), '[EOS]', NEMO_CHAR.closure() + ) + + rewrite_add_separator_fst = pynini.compose(rewrite_na_fst, rewrite_n_fst) + ten_thousand = pynini.string_map([("ibihumbi_icumi", "10")]) + ten = pynini.string_map([("icumi", "10")]) + digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) + digits_for_thousands = pynini.string_file(get_abs_path("data/cardinal/digits_for_thousands.tsv")) + digits_millions_trillions = pynini.string_file(get_abs_path("data/cardinal/digits_millions_trillions.tsv")) + tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) + tens_for_ends = pynini.string_map([("icumi", "1")]) | tens + tens_for_beginnings = pynini.string_map([("cumi", "1")]) | tens + hundreds = pynini.string_file(get_abs_path("data/cardinal/hundreds.tsv")) + thousands = pynini.string_file(get_abs_path("data/cardinal/thousands.tsv")) + tens_of_thousands = pynini.string_file(get_abs_path("data/cardinal/tens_of_thousands.tsv")) + hundreds_of_thousands = pynini.string_file(get_abs_path("data/cardinal/hundreds_of_thousands.tsv")) + millions = pynini.string_file(get_abs_path("data/cardinal/millions.tsv")) + tens_of_millions = pynini.string_file(get_abs_path("data/cardinal/tens_of_millions.tsv")) + hundreds_of_millions = pynini.string_file(get_abs_path("data/cardinal/hundreds_of_millions.tsv")) + trillions = pynini.string_file(get_abs_path("data/cardinal/trillions.tsv")) + tens_of_trillions = pynini.string_file(get_abs_path("data/cardinal/tens_of_trillions.tsv")) + hundreds_of_trillions = pynini.string_file(get_abs_path("data/cardinal/hundreds_of_trillions.tsv")) + + THREE_ZEROS = "000" + FOUR_ZEROS = "0000" + FIVE_ZEROS = "00000" + SIX_ZEROS = "000000" + SEVEN_ZEROS = "0000000" + EIGHT_ZEROS = "00000000" + NINE_ZEROS = "000000000" + + zero = pynini.string_map([("zeru", "0")]) + rewrite_remove_comma_fst = pynini.cdrewrite( + pynini.cross(",", ""), pynini.union(NEMO_DIGIT), pynini.union(NEMO_DIGIT), NEMO_CHAR.closure() + ) + single_digits_graph = pynini.invert(digits | zero) + single_digits_graph = single_digits_graph + pynini.closure(insert_space + single_digits_graph) + remove_comma = rewrite_remove_comma_fst @ single_digits_graph + + graph_tens_ends = tens_for_ends + pynutil.delete(" ") + digits | tens_for_ends + pynutil.insert("0") + graph_tens_starts = tens_for_beginnings + pynutil.delete(" ") + digits | tens_for_beginnings + pynutil.insert( + "0" + ) + + graph_tens_for_thousands = tens_for_beginnings + pynutil.delete( + " " + ) + digits_for_thousands | tens_for_beginnings + pynutil.insert("0") + + graph_tens_for_millions_trillions = tens_for_beginnings + pynutil.delete( + " " + ) + digits_millions_trillions | tens_for_beginnings + pynutil.insert("0") + graph_hundreds = ( + hundreds + pynutil.delete(" ") + graph_tens_ends + | hundreds + pynutil.insert("00") + | hundreds + pynutil.delete(" ") + pynutil.insert("0") + digits + ) + graph_thousands = ( + thousands + pynutil.delete(" ") + graph_hundreds + | thousands + pynutil.insert(THREE_ZEROS) + | thousands + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_ends + | thousands + pynutil.delete(" ") + pynutil.insert("00") + digits + ) + + graph_ten_thousand_and_hundreds = ( + ten_thousand + pynutil.insert(THREE_ZEROS) + | ten_thousand + pynutil.delete(" ") + graph_hundreds + | ten_thousand + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_ends + | ten_thousand + pynutil.delete(" ") + pynutil.insert("00") + digits + ) + prefix_tens_of_thousands = tens_of_thousands + pynutil.delete(" ") + digits_for_thousands + graph_tens_of_thousands = ( + pynutil.add_weight(graph_ten_thousand_and_hundreds, weight=-0.1) + | prefix_tens_of_thousands + pynutil.delete(" ") + graph_hundreds + | prefix_tens_of_thousands + pynutil.insert(THREE_ZEROS) + | prefix_tens_of_thousands + pynutil.delete(" ") + pynutil.insert("0") + graph_hundreds + | prefix_tens_of_thousands + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_ends + | prefix_tens_of_thousands + pynutil.delete(" ") + pynutil.insert("00") + digits + ) + + prefix_hundreds_of_thousands = hundreds_of_thousands + pynutil.delete(" ") + graph_tens_for_thousands + graph_hundreds_of_thousands = ( + hundreds_of_thousands + pynutil.insert(FIVE_ZEROS) + | prefix_hundreds_of_thousands + pynutil.insert(THREE_ZEROS) + | prefix_hundreds_of_thousands + pynutil.delete(" ") + graph_hundreds + | pynutil.add_weight( + prefix_hundreds_of_thousands + pynutil.delete(" ") + pynutil.insert("00") + digits, weight=-0.1 + ) + | prefix_hundreds_of_thousands + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_for_thousands + ) + + graph_millions = ( + millions + pynutil.delete(" ") + graph_hundreds_of_thousands + | millions + pynutil.insert(SIX_ZEROS) + | millions + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_of_thousands + | millions + pynutil.delete(" ") + pynutil.insert("00") + graph_thousands + | millions + pynutil.delete(" ") + pynutil.insert(THREE_ZEROS) + graph_hundreds + | millions + pynutil.delete(" ") + pynutil.insert(FOUR_ZEROS) + graph_tens_ends + | millions + pynutil.delete(" ") + pynutil.insert(FIVE_ZEROS) + digits + ) + + prefix_tens_of_millions = tens_of_millions + pynutil.delete(" ") + digits_millions_trillions + graph_tens_of_millions = ( + prefix_tens_of_millions + pynutil.delete(" ") + graph_hundreds_of_thousands + | prefix_tens_of_millions + pynutil.delete(" ") + pynutil.insert(SIX_ZEROS) + | prefix_tens_of_millions + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_of_thousands + | prefix_tens_of_millions + pynutil.delete(" ") + pynutil.insert(THREE_ZEROS) + graph_hundreds + | prefix_tens_of_millions + pynutil.delete(" ") + pynutil.insert(FOUR_ZEROS) + graph_tens_ends + | tens_of_millions + pynutil.delete(" ") + pynutil.insert(FIVE_ZEROS) + graph_tens_ends + | prefix_tens_of_millions + pynutil.delete(" ") + pynutil.insert(FIVE_ZEROS) + digits + ) + + prefix_hundreds_of_millions = hundreds_of_millions + pynutil.delete(" ") + graph_tens_for_millions_trillions + graph_hundreds_of_millions = ( + prefix_hundreds_of_millions + pynutil.delete(" ") + graph_hundreds_of_thousands + | prefix_hundreds_of_millions + pynutil.insert(SIX_ZEROS) + | prefix_hundreds_of_millions + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_of_thousands + | prefix_hundreds_of_millions + pynutil.delete(" ") + pynutil.insert("00") + graph_thousands + | prefix_hundreds_of_millions + pynutil.delete(" ") + pynutil.insert(THREE_ZEROS) + graph_hundreds + | prefix_hundreds_of_millions + pynutil.delete(" ") + pynutil.insert(FOUR_ZEROS) + graph_tens_ends + ) + + graph_trillions = ( + trillions + pynutil.delete(" ") + graph_hundreds_of_millions + | trillions + pynutil.insert(NINE_ZEROS) + | trillions + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_of_millions + | trillions + pynutil.delete(" ") + pynutil.insert("00") + graph_millions + | trillions + pynutil.delete(" ") + pynutil.insert(THREE_ZEROS) + graph_hundreds_of_thousands + | trillions + pynutil.delete(" ") + pynutil.insert(FOUR_ZEROS) + graph_tens_of_thousands + | trillions + pynutil.delete(" ") + pynutil.insert(FIVE_ZEROS) + graph_thousands + | trillions + pynutil.delete(" ") + pynutil.insert(SIX_ZEROS) + graph_hundreds + | trillions + pynutil.delete(" ") + pynutil.insert(SEVEN_ZEROS) + graph_tens_ends + | trillions + pynutil.delete(" ") + pynutil.insert(EIGHT_ZEROS) + digits + ) + + prefix_tens_of_trillions = tens_of_trillions + pynutil.delete(" ") + digits_millions_trillions + graph_tens_of_trillions = ( + prefix_tens_of_trillions + pynutil.delete(" ") + graph_hundreds_of_millions + | prefix_tens_of_trillions + pynutil.insert(NINE_ZEROS) + | prefix_tens_of_trillions + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_of_millions + | prefix_tens_of_trillions + pynutil.delete(" ") + pynutil.insert("00") + graph_millions + | prefix_tens_of_trillions + + pynutil.delete(" ") + + pynutil.insert(THREE_ZEROS) + + graph_hundreds_of_thousands + | prefix_tens_of_trillions + pynutil.delete(" ") + pynutil.insert(FOUR_ZEROS) + graph_tens_of_thousands + | prefix_tens_of_trillions + pynutil.delete(" ") + pynutil.insert(FIVE_ZEROS) + graph_thousands + | prefix_tens_of_trillions + pynutil.delete(" ") + pynutil.insert(SIX_ZEROS) + graph_hundreds + | prefix_tens_of_trillions + pynutil.delete(" ") + pynutil.insert(SEVEN_ZEROS) + graph_tens_ends + | prefix_tens_of_trillions + pynutil.delete(" ") + pynutil.insert(EIGHT_ZEROS) + digits + ) + + prefix_hundreds_of_trillions = hundreds_of_trillions + pynutil.delete(" ") + graph_tens_for_millions_trillions + graph_hundreds_of_trillions = ( + prefix_hundreds_of_trillions + pynutil.delete(" ") + graph_hundreds_of_millions + | prefix_hundreds_of_trillions + pynutil.insert(NINE_ZEROS) + | prefix_hundreds_of_trillions + pynutil.delete(" ") + pynutil.insert("0") + graph_tens_of_millions + | prefix_hundreds_of_trillions + pynutil.delete(" ") + pynutil.insert("00") + graph_millions + | prefix_hundreds_of_trillions + + pynutil.delete(" ") + + pynutil.insert(THREE_ZEROS) + + graph_hundreds_of_thousands + | prefix_hundreds_of_trillions + pynutil.delete(" ") + pynutil.insert(FOUR_ZEROS) + graph_tens_of_thousands + | prefix_hundreds_of_trillions + pynutil.delete(" ") + pynutil.insert(FIVE_ZEROS) + graph_thousands + | prefix_hundreds_of_trillions + pynutil.delete(" ") + pynutil.insert(SIX_ZEROS) + graph_hundreds + | prefix_hundreds_of_trillions + pynutil.delete(" ") + pynutil.insert(SEVEN_ZEROS) + graph_tens_ends + ) + + graph_all = ( + graph_hundreds_of_trillions + | graph_tens_of_trillions + | graph_trillions + | graph_hundreds_of_millions + | graph_tens_of_millions + | graph_millions + | graph_hundreds_of_thousands + | graph_tens_of_thousands + | graph_thousands + | graph_hundreds + | pynutil.add_weight(ten, weight=-0.1) + | graph_tens_starts + | digits + | pynini.cross("zeru", "0") + ) + + inverted_graph_all = pynini.compose(pynini.invert(graph_all), rewrite_add_separator_fst) + inverted_graph_all = pynini.compose(inverted_graph_all, remove_extra_space_fst) + inverted_graph_all = pynini.compose(inverted_graph_all, remove_trailing_space_fst) + inverted_graph_all = pynini.compose(inverted_graph_all, remove_underscore_fst) | pynutil.add_weight( + remove_comma, 0.0001 + ) + + inverted_graph_all = inverted_graph_all.optimize() + final_graph = pynutil.insert("integer: \"") + inverted_graph_all + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph diff --git a/nemo_text_processing/text_normalization/rw/taggers/time.py b/nemo_text_processing/text_normalization/rw/taggers/time.py new file mode 100644 index 000000000..0caf4f7d5 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/taggers/time.py @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.rw.graph_utils import GraphFst +from nemo_text_processing.text_normalization.rw.utils import get_abs_path + + +class TimeFst(GraphFst): + def __init__(self): + super().__init__(name="time", kind="classify") + + hours = pynini.string_file(get_abs_path("data/time/hours.tsv")) + + minutes = pynini.string_file(get_abs_path("data/time/minutes.tsv")) + + final_graph = ( + pynutil.insert("hours:\"") + + hours + + pynutil.insert("\"") + + pynutil.delete(":") + + pynutil.insert(" minutes:\"") + + minutes + + pynutil.insert("\"") + ) + final_graph = self.add_tokens(final_graph) + + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/rw/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/rw/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..01ec1e370 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/taggers/tokenize_and_classify.py @@ -0,0 +1,78 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.en.taggers.word import WordFst +from nemo_text_processing.text_normalization.rw.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.rw.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.rw.taggers.time import TimeFst +from nemo_text_processing.text_normalization.rw.taggers.whitelist import WhiteListFst + + +class ClassifyFst(GraphFst): + def __init__( + self, + input_case: str, + cache_dir: str = None, + overwrite_cache: bool = False, + deterministic: bool = True, + whitelist: str = None, + ): + super().__init__(name='tokenize_and_classify', kind='classify', deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "rw_tn_tokenize_and_classify.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + print("FAR file: ", far_file) + self.fst = pynini.Far(far_file, mode="r")["TOKENIZE_AND_CLASSIFY"] + else: + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + time_graph = TimeFst().fst + punctuation = PunctuationFst() + punct_graph = punctuation.fst + + word_graph = WordFst(punctuation=punctuation).fst + + whitelist_graph = WhiteListFst().fst + classify = ( + pynutil.add_weight(time_graph, 1.05) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(word_graph, 1.50) + | pynutil.add_weight(whitelist_graph, 1.01) + ) + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + ) + + graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) + graph = delete_space + graph + delete_space + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"TOKENIZE_AND_CLASSIFY": self.fst}) diff --git a/nemo_text_processing/text_normalization/rw/taggers/whitelist.py b/nemo_text_processing/text_normalization/rw/taggers/whitelist.py new file mode 100644 index 000000000..382243d26 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/taggers/whitelist.py @@ -0,0 +1,32 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.rw.graph_utils import GraphFst +from nemo_text_processing.text_normalization.rw.utils import get_abs_path + +transliterations = pynini.string_file(get_abs_path("data/whitelist/kinya_transliterations.tsv")) + + +class WhiteListFst(GraphFst): + def __init__(self): + super().__init__(name="whitelist", kind="classify") + + whitelist = transliterations + graph = pynutil.insert("name: \"") + whitelist + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/rw/utils.py b/nemo_text_processing/text_normalization/rw/utils.py new file mode 100644 index 000000000..460596bca --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/text_normalization/rw/verbalizers/__init__.py b/nemo_text_processing/text_normalization/rw/verbalizers/__init__.py new file mode 100644 index 000000000..9c4313114 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/verbalizers/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/rw/verbalizers/time.py b/nemo_text_processing/text_normalization/rw/verbalizers/time.py new file mode 100644 index 000000000..50c0f71a2 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/verbalizers/time.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.rw.graph_utils import NEMO_CHAR, GraphFst, delete_space + + +class VerbalizeTimeFst(GraphFst): + def __init__(self): + super().__init__(name="time", kind="verbalize") + hour = ( + pynutil.delete("hours:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_CHAR) + + pynutil.delete("\"") + ) + + graph = hour + delete_tokens = self.delete_tokens(graph) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/rw/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/rw/verbalizers/verbalize.py new file mode 100644 index 000000000..267215145 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/verbalizers/verbalize.py @@ -0,0 +1,29 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from nemo_text_processing.text_normalization.en.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.rw.graph_utils import GraphFst +from nemo_text_processing.text_normalization.rw.verbalizers.time import VerbalizeTimeFst + + +class VerbalizeFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + time = VerbalizeTimeFst().fst + + graph = cardinal_graph | time + self.fst = graph diff --git a/nemo_text_processing/text_normalization/rw/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/rw/verbalizers/verbalize_final.py new file mode 100644 index 000000000..d39210ff5 --- /dev/null +++ b/nemo_text_processing/text_normalization/rw/verbalizers/verbalize_final.py @@ -0,0 +1,53 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst +from nemo_text_processing.text_normalization.rw.graph_utils import GraphFst, delete_space, generator_main +from nemo_text_processing.text_normalization.rw.verbalizers.verbalize import VerbalizeFst + + +class VerbalizeFinalFst(GraphFst): + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False, deterministic: bool = True): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"rw_tn_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_space) + graph + delete_space + + self.fst = graph + + if far_file: + generator_main(far_file, {"ALL": self.fst, 'REDUP': pynini.accep("REDUP")}) diff --git a/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv b/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv index 33e616872..207223772 100644 --- a/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv +++ b/nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv @@ -1,14 +1,13 @@ . čuokkis : duppalčuokkis -- goallossáhcu +- sáhcu ! čuorvvasmearka ; beallečuokkis ' apostrofa / finjusáhcu ? gažaldatmearka , rihkku -_ vuollesárggis -! čuorvvasmearka +_ deattuhusmearka # fáddágilkor $ dollár % proseanta diff --git a/nemo_text_processing/text_normalization/se/data/inflection/case_suffixes.tsv b/nemo_text_processing/text_normalization/se/data/inflection/case_suffixes.tsv new file mode 100644 index 000000000..8494cf569 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/inflection/case_suffixes.tsv @@ -0,0 +1,9 @@ +nom_pl t +gen_pl id +ill_sg i +ill_pl ide +loc_sg s +loc_pl in +com_sg in +com_pl iguin +ess n diff --git a/nemo_text_processing/text_normalization/se/data/inflection/ordinal_case_suffixes.tsv b/nemo_text_processing/text_normalization/se/data/inflection/ordinal_case_suffixes.tsv new file mode 100644 index 000000000..893204555 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/inflection/ordinal_case_suffixes.tsv @@ -0,0 +1,12 @@ +nom_pl dat +gen_sg da +acc_sg da +gen_pl diid +acc_pl diid +ill_sg dii +ill_pl diidda +loc_sg dis +loc_pl diin +com_sg diin +com_pl diiguin +ess din diff --git a/nemo_text_processing/text_normalization/se/data/measure/unit_genitive.tsv b/nemo_text_processing/text_normalization/se/data/measure/unit_genitive.tsv new file mode 100644 index 000000000..90b599b9e --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/measure/unit_genitive.tsv @@ -0,0 +1,2 @@ +m mehtera +km kilomehtera diff --git a/nemo_text_processing/text_normalization/se/data/measure/unit_rate.tsv b/nemo_text_processing/text_normalization/se/data/measure/unit_rate.tsv new file mode 100644 index 000000000..50ee52b57 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/measure/unit_rate.tsv @@ -0,0 +1,4 @@ +km/d kilomehtera diimmus +km/dii. kilomehtera diimmus +km/h kilomehtera diimmus +m/s mehtera sekunddas diff --git a/nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv b/nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv index 65a5dc041..b7d95ecc9 100644 --- a/nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv +++ b/nemo_text_processing/text_normalization/se/data/measure/unit_simple.tsv @@ -9,5 +9,43 @@ cm3 guđahassentimehter mm³ guđahasmillimehter mm3 guđahasmillimehter kg kilográmma +˚C celsiusgráda +°C celsiusgráda +°F Fahrenheit-gráda +A ampera +cal. kaloriija +cg centigrámma +cl centilihtar +dB desibel +dg desigrámma +dl desilihtar +dm desimehter g. grámma +g grámma +hg hektográmma +hl hektolihtar +Hz hertz +J joule +K Kelvin-gráda +kB kilobyte +km kilomehter +kV kilovolta +kVA kilovoltampere +kW kilowatt +kWh kilowattdiibmu +l lihtar +m mehter +m² njealjehasmehter +MB megabyte +mg milligrámma +MHz megahertz +ml millilihtar +mV millivolt +MW megawatt +mW milliwatt +N newton t. tonna +t tonna +V volt +W watt +yd yard diff --git a/nemo_text_processing/text_normalization/se/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/se/data/money/currency_major.tsv index 14ff303f2..773dca204 100644 --- a/nemo_text_processing/text_normalization/se/data/money/currency_major.tsv +++ b/nemo_text_processing/text_normalization/se/data/money/currency_major.tsv @@ -1,6 +1,18 @@ sek ruoŧa ruvdno nok norgga ruvdno kr ruvdno +bfr Belgia franc +Dkr Dánmárku ruvdno +DKK Dánmárku ruvdno +FIM Suoma márki +fr Fránkriikka franc +mk márki +Nkr norgga ruvdno +NOK norgga ruvdno +pta pesetas +rbl rubel +SEK ruoŧa ruvdno +sfr Šveicca franc £ punda eur euro € euro diff --git a/nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv b/nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv index d0a0d8c56..085f8c482 100644 --- a/nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv +++ b/nemo_text_processing/text_normalization/se/data/money/currency_major_gen.tsv @@ -1,6 +1,9 @@ sek ruoŧa ruvnno nok norgga ruvnno kr ruvnno +Nkr norgga ruvnno +NOK norgga ruvnno +SEK ruoŧa ruvnno £ punda eur euro € euro diff --git a/nemo_text_processing/text_normalization/se/data/numbers/compound_digit.tsv b/nemo_text_processing/text_normalization/se/data/numbers/compound_digit.tsv new file mode 100644 index 000000000..429acd693 --- /dev/null +++ b/nemo_text_processing/text_normalization/se/data/numbers/compound_digit.tsv @@ -0,0 +1,8 @@ +2 guovtte +3 golmma +4 njealje +5 viđa +6 guđa +7 čieža +8 gávcci +9 ovcce diff --git a/nemo_text_processing/text_normalization/se/data/whitelist.tsv b/nemo_text_processing/text_normalization/se/data/whitelist.tsv index d2e304dcd..7799c48f8 100644 --- a/nemo_text_processing/text_normalization/se/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/se/data/whitelist.tsv @@ -1,4 +1,80 @@ tel. telefonnummár +abstr. abstrákta +adv. advokáhtta +affekt. affektiiva +anat. anatomiija +art. artihkkal +biol. biologiija +bot. botanihkka +čuovv. čuovvovaš +dáb. dábálaš +dár. dárogiella +dept. departemeanta +dii. diibmu +dir. direktevra +doaimm. doaimmaheaddji +duođ. duođaštus +eaŋg. eaŋgalasgiella +eŋg. eŋgelasgiella +euf. eufemisttalaš +eur. eurohpálaš +fig. figuratiiva +fr. fránskkagiella +fuom. fuomáš +gč. geahča +geogr. geográfalaš +geom. geometriija +govv. govvideaddji +ing. ingenevra +jur. juridihkalaš +kand. kandidáhtta +kap. kapihtal +konkr. konkrehta +korr. korrektuvra +lab. laboratoriija +lingv. lingvistihkka +mat. matematihkka +matem. matematihkka +med. medisiidna +milj. miljovdna +miner. minerála +mrd. miljárda +mus. musihkka +neg. negatiiva +num. numerála +om. ovdamearkan +omd. ovdamearkka dihte +par. paragráfa +pers. persovdna +pos. positiiva +psyk. psykologiija +rieg. riegádan +ráđđ. ráđđehus +rek. rekommanderejuvvon +rel. religiovdna +relat. relatiiva +relig. religiovdna +res. resolušuvdna +ruoŧ. ruoŧagiella +s. siidu +sám. sámegiella +sápm. sápmelaš +siviling. sivilingenevra +skand. skandinávalaš +st. stašuvdna +stip. stipendiáhtta +stud. studeanta +su. sullii +suom. suomagiella +tab. tabealla +tekn. teknihkka +tii. tiibma +tlf. telefovdna +tlg. telegrámma +vol. volum +vrd. veardit +vást. vástádus +vulg. vulgára EU Eurohpauniovdna EU:a Eurohpauniovnna EU:i Eurohpauniovdnii diff --git a/nemo_text_processing/text_normalization/se/graph_utils.py b/nemo_text_processing/text_normalization/se/graph_utils.py index 5c5dbe82a..5fc44a806 100644 --- a/nemo_text_processing/text_normalization/se/graph_utils.py +++ b/nemo_text_processing/text_normalization/se/graph_utils.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import delete_space from pynini.lib import byte, pynutil -_ALPHA_UPPER = "АÁBCČDĐEFGHIJKLMNŊOPRSŠTŦUVZŽÆØÅÄÖ" -_ALPHA_LOWER = "аábcčdđefghijklmnŋoprsštŧuvzžæøåäö" +from nemo_text_processing.text_normalization.en.graph_utils import delete_space + +_ALPHA_UPPER = "AÁBCČDĐEFGHIJKLMNŊOPQRSŠTŦUVWXYZŽÆØÅÄÖ" +_ALPHA_LOWER = "aábcčdđefghijklmnŋopqrsštŧuvwxyzžæøåäö" TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(_ALPHA_UPPER, _ALPHA_LOWER)]) TO_UPPER = pynini.invert(TO_LOWER) diff --git a/nemo_text_processing/text_normalization/se/taggers/abbreviation.py b/nemo_text_processing/text_normalization/se/taggers/abbreviation.py index a8b624543..821647150 100644 --- a/nemo_text_processing/text_normalization/se/taggers/abbreviation.py +++ b/nemo_text_processing/text_normalization/se/taggers/abbreviation.py @@ -14,9 +14,11 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space from nemo_text_processing.text_normalization.se.graph_utils import SE_UPPER -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.se.utils import get_abs_path class AbbreviationFst(GraphFst): @@ -41,12 +43,16 @@ def __init__(self, whitelist: 'pynini.FstLike', deterministic: bool = True): # ABC -> A B C graph |= SE_UPPER + pynini.closure(insert_space + SE_UPPER, 1) + case_suffix = pynini.project(pynini.string_file(get_abs_path("data/inflection/case_suffixes.tsv")), "output") + graph += pynini.closure(pynini.accep(":") + case_suffix, 0, 1) + # exclude words that are included in the whitelist if whitelist is not None: graph = pynini.compose( pynini.difference(pynini.project(graph, "input"), pynini.project(whitelist.graph, "input")), graph ) - graph = pynutil.insert("value: \"") + graph.optimize() + pynutil.insert("\"") + self.graph = graph.optimize() + graph = pynutil.insert("value: \"") + self.graph + pynutil.insert("\"") graph = self.add_tokens(graph) self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/cardinal.py b/nemo_text_processing/text_normalization/se/taggers/cardinal.py index 684f1fa21..ea9ab3b73 100644 --- a/nemo_text_processing/text_normalization/se/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/se/taggers/cardinal.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -22,8 +24,13 @@ delete_space, ) from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA, make_spacer -from nemo_text_processing.text_normalization.se.utils import CASE_KEYS, get_abs_path, load_case_forms, load_labels -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.se.utils import ( + CASE_ALIASES, + CASE_KEYS, + get_abs_path, + load_case_forms, + load_labels, +) def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': @@ -37,7 +44,7 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': Returns: fst: A pynini.FstLike object """ - exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string cardinal_separator = pynini.union(NEMO_SPACE, ".") @@ -134,11 +141,13 @@ def get_digit_cased_fst(bare=True, deterministic=deterministic): if k == "nom_sg": teens_cased_fst[k] = pynutil.delete("1") + digits_cased_fst[k] + pynutil.insert("nuppelohkái") else: - teens_cased_fst[k] = pynutil.delete("1") + digits_cased_fst[k] + pynutil.insert(f"nuppe{logi_cased[k]}") + teens_cased_fst[k] = ( + pynutil.delete("1") + digits_cased_fst["nom_sg"] + pynutil.insert(f"nuppe{logi_cased[k]}") + ) if not deterministic: if k in ["nom_pl", "gen_pl", "acc_pl", "loc_pl"]: dbc = digits_bare_cased_fst[k] - teens_cased_fst[k] = pynutil.delete("1") + dbc + pynutil.insert(f"nuppe{logi_cased[k]}") + teens_cased_fst[k] |= pynutil.delete("1") + dbc + pynutil.insert(f"nuppe{logi_cased[k]}") if k == "ess": teens_cased_fst["ess"] |= pynutil.delete("1") + digits_cased_fst["ess"] + nuppelogin teens_cased_fst["ess"] |= pynutil.delete("1") + digits_cased_fst["nom_pl"] + nuppelogin @@ -246,7 +255,7 @@ def select_tens(tens_cased): bare_thousand = pynini.cross("000", duhat_cased[k]) prefix_digit = (NEMO_DIGIT - "1") @ digits_prefix_cased_fst[k] prefix_digit |= pynutil.delete("1") - if not deterministic and k == "sg_gen": + if not deterministic and k == "gen_sg": bare_thousand |= pynini.cross("000", "duhát") bare_thousands_fst[k] = prefix_digit + spacer + bare_thousand thousands_fst = {} @@ -414,7 +423,7 @@ def __init__(self, deterministic: bool = True): self.graph = ( ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) @@ -426,6 +435,50 @@ def __init__(self, deterministic: bool = True): self.graph = filter_punctuation(self.graph).optimize() + compound_digit = pynini.string_file(get_abs_path("data/numbers/compound_digit.tsv")) + compound_teen = pynutil.delete("1") + digit + pynutil.insert("nuppeloh") + if not deterministic: + compound_teen |= pynutil.delete("1") + digit + pynutil.insert("nuppelot") + self.compound = pynini.union( + compound_digit, + pynini.cross("10", "logi"), + compound_teen, + digits_no_one + pynini.cross("0", "logi"), + digits_no_one + pynutil.insert("logi") + compound_digit, + ).optimize() + + cased = build_cased_number_fsts(deterministic) + self.graphs = {"nom_sg": self.graph} + for case in CASE_KEYS: + case_graph = pynini.union( + cased["zero"][case], + cased["digits"][case], + cased["two_digit_cased_fsts"][case], + cased["three_digit_cased_fsts"][case], + cased["bare_hundreds"][case], + cased["bare_thousands"][case], + cased["thousands"][case], + ).optimize() + self.graphs[case] = filter_punctuation(case_graph).optimize() + + for alias, case in CASE_ALIASES.items(): + self.graphs[alias] = self.graphs[case] + self.graph_dict = self.graphs + + # Riektačállinrávvagat (Sámediggi, revised 2019), pp. 60–61: + # "Go vuođđologuide laktá sojahangehčosiid, de čállá logu maŋŋái + # duppalčuoggá ja dan maŋŋái fas kásusgehčosa." + case_suffixes = load_labels(get_abs_path("data/inflection/case_suffixes.tsv")) + explicit_case_graphs = [] + for case, suffix in case_suffixes: + explicit_case_graphs.append(self.graphs[case] + pynutil.delete(":") + pynutil.delete(suffix)) + self.graph_suffixed = pynini.union(*explicit_case_graphs).optimize() + + self.graph |= self.graph_suffixed + if not deterministic: + self.graph |= pynini.union(*self.graphs.values()) + self.graph = self.graph.optimize() + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") diff --git a/nemo_text_processing/text_normalization/se/taggers/date.py b/nemo_text_processing/text_normalization/se/taggers/date.py index e6c830381..2fe1996b3 100644 --- a/nemo_text_processing/text_normalization/se/taggers/date.py +++ b/nemo_text_processing/text_normalization/se/taggers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -23,7 +25,6 @@ ) from nemo_text_processing.text_normalization.se.graph_utils import TO_LOWER from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels -from pynini.lib import pynutil delete_leading_zero = (pynutil.delete("0") | (NEMO_DIGIT - "0")) + NEMO_DIGIT diff --git a/nemo_text_processing/text_normalization/se/taggers/decimal.py b/nemo_text_processing/text_normalization/se/taggers/decimal.py index 00b436da0..27ae28087 100644 --- a/nemo_text_processing/text_normalization/se/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/se/taggers/decimal.py @@ -14,14 +14,18 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.se.utils import get_abs_path -from pynini.lib import pynutil quantities = pynini.string_file(get_abs_path("data/numbers/millions.tsv")) -def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_thousand: 'pynini.FstLike',) -> 'pynini.FstLike': +def get_quantity( + decimal: 'pynini.FstLike', + cardinal_up_to_thousand: 'pynini.FstLike', +) -> 'pynini.FstLike': """ Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral, e.g. 1 miljárda -> integer_part: "okta" quantity: "miljárda" @@ -31,7 +35,14 @@ def get_quantity(decimal: 'pynini.FstLike', cardinal_up_to_thousand: 'pynini.Fst decimal: decimal FST cardinal_up_to_hundred: cardinal FST """ - nom_to_gen_endings = pynini.string_map(("on", "ovnna"), ("ovdna", "ovnna"), ("árda", "árdda",)) + nom_to_gen_endings = pynini.string_map( + ("on", "ovnna"), + ("ovdna", "ovnna"), + ( + "árda", + "árdda", + ), + ) quantities_gen = quantities @ pynini.cdrewrite(nom_to_gen_endings, "", "[EOS]", NEMO_SIGMA) res = ( diff --git a/nemo_text_processing/text_normalization/se/taggers/electronic.py b/nemo_text_processing/text_normalization/se/taggers/electronic.py index 528b1838e..e68ce4be6 100644 --- a/nemo_text_processing/text_normalization/se/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/se/taggers/electronic.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels -from pynini.lib import pynutil common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))] symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))] diff --git a/nemo_text_processing/text_normalization/se/taggers/fraction.py b/nemo_text_processing/text_normalization/se/taggers/fraction.py index b48f65e54..f632d2ef8 100644 --- a/nemo_text_processing/text_normalization/se/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/se/taggers/fraction.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.se.graph_utils import ensure_space from nemo_text_processing.text_normalization.se.utils import get_abs_path -from pynini.lib import pynutil class FractionFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/se/taggers/measure.py b/nemo_text_processing/text_normalization/se/taggers/measure.py index 443d9e889..8b6cc407e 100644 --- a/nemo_text_processing/text_normalization/se/taggers/measure.py +++ b/nemo_text_processing/text_normalization/se/taggers/measure.py @@ -1,5 +1,5 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, 2026, Jim O'Regan for Språkbanken Tal # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,199 +14,52 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import ( - NEMO_NON_BREAKING_SPACE, - NEMO_SIGMA, - GraphFst, - convert_space, - delete_space, - delete_zero_or_one_space, -) -from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA, TO_LOWER -from nemo_text_processing.text_normalization.se.utils import get_abs_path from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_zero_or_one_space +from nemo_text_processing.text_normalization.se.utils import get_abs_path + class MeasureFst(GraphFst): - """ - Finite state transducer for classifying measure, suppletive aware, e.g. - -12kg -> measure { negative: "true" cardinal { integer: "guoktenuppelohkái" } units: "kilográmma" } - 1kg -> measure { cardinal { integer: "okta" } units: "kilográmma" } - ,5kg -> measure { decimal { fractional_part: "guhtta" } units: "kilográmma" } - - Args: - cardinal: CardinalFst - decimal: DecimalFst - fraction: FractionFst - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + """Classifies documented Northern Sámi cardinal measure expressions.""" - def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True): + def __init__(self, cardinal: GraphFst, decimal=None, fraction=None, deterministic: bool = True): super().__init__(name="measure", kind="classify", deterministic=deterministic) - cardinal_graph = cardinal.graph - unit_simple = pynini.string_file(get_abs_path("data/measure/unit_simple.tsv")) - simple_endings = pynini.string_file(get_abs_path("data/inflection/simple.tsv")) - graph_simple = unit_simple | unit_simple + simple_endings - - graph_plurals = pynini.string_file(get_abs_path("data/measure/unit_plural.tsv")) - greek_lower = pynini.string_file(get_abs_path("data/measure/greek_lower.tsv")) - greek_upper = pynutil.insert("stuorra ") + pynini.string_file(get_abs_path("data/measure/greek_upper.tsv")) - greek = greek_lower | greek_upper - - graph_unit |= pynini.compose( - pynini.closure(TO_LOWER, 1) + (SE_ALPHA | TO_LOWER) + pynini.closure(SE_ALPHA | TO_LOWER), graph_unit + unit_nominative = pynini.string_file(get_abs_path("data/measure/unit_simple.tsv")) + unit_genitive = pynini.string_file(get_abs_path("data/measure/unit_genitive.tsv")) + unit_rate = pynini.string_file(get_abs_path("data/measure/unit_rate.tsv")) + + one = pynini.accep("1") @ cardinal.graph + non_one = pynini.difference(pynini.project(cardinal.graph, "input"), "1") @ cardinal.graph + + def cardinal_token(graph): + return pynutil.insert('cardinal { integer: "') + graph + pynutil.insert('" } ') + + def unit_token(graph): + return pynutil.insert('units: "') + graph + pynutil.insert('"') + + separator = delete_zero_or_one_space + singular = cardinal_token(one) + separator + unit_token(unit_nominative) + governed = cardinal_token(non_one) + separator + unit_token(unit_genitive) + rate = cardinal_token(cardinal.graph) + separator + unit_token(unit_rate) + + # Riektačállinrávvagat (Sámediggi, revised 2019), p. 57: + # "ovcce- ja guoktenuppelohjahkáččat" corresponds to + # "9- ja 12-jahkásaččat". + age_adjective = pynini.cross("jahkásaš", "jahkásaš") | pynini.cross("jahkásaččat", "jahkáččat") + if not deterministic: + age_adjective |= pynini.cross("jahkásaččat", "jahkásaččat") + age = cardinal.compound + pynutil.delete("-") + age_adjective + age_token = pynutil.insert('name: "') + age + pynutil.insert('"') + + if not deterministic: + governed |= cardinal_token(non_one) + separator + unit_token(unit_nominative) + + self.fst = (self.add_tokens(singular | governed | rate) | age_token).optimize() + self.graph = ( + one + separator + pynutil.insert(" ") + unit_nominative + | non_one + separator + pynutil.insert(" ") + unit_genitive + | cardinal.graph + separator + pynutil.insert(" ") + unit_rate + | age ).optimize() - - graph_unit_plural = convert_space(graph_unit @ graph_plurals) - graph_unit = convert_space(graph_unit) - # placeholder: probably needs to be a file - graph_unit_gen = graph_unit - - optional_graph_negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) - - graph_unit2 = ( - pynutil.delete("/") - + delete_zero_or_one_space - + pynutil.insert(NEMO_NON_BREAKING_SPACE) - + graph_unit_gen - + pynutil.insert(" mielde") - ) - - optional_graph_unit2 = pynini.closure( - delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, 0, 1, - ) - - unit_plural = ( - pynutil.insert("units: \"") - + (graph_unit_plural + optional_graph_unit2 | graph_unit2) - + pynutil.insert("\"") - ) - - unit_singular = ( - pynutil.insert("units: \"") + (graph_unit + optional_graph_unit2 | graph_unit2) + pynutil.insert("\"") - ) - - subgraph_decimal = ( - pynutil.insert("decimal { ") - + optional_graph_negative - + decimal.final_graph_wo_negative_en - + delete_space - + pynutil.insert(" } ") - + unit_plural - ) - - # support radio FM/AM - subgraph_decimal |= ( - pynutil.insert("decimal { ") - + decimal.final_graph_wo_negative - + delete_space - + pynutil.insert(" } ") - + pynutil.insert("units: \"") - + pynini.union("AM", "FM") - + pynutil.insert("\"") - ) - - subgraph_cardinal = ( - pynutil.insert("cardinal { ") - + optional_graph_negative - + pynutil.insert("integer: \"") - + (NEMO_SIGMA @ cardinal_graph) - + delete_space - + pynutil.insert("\"") - + pynutil.insert(" } ") - + unit_plural - ) - self.subgraph_cardinal = subgraph_cardinal - - unit_graph = ( - pynutil.insert("cardinal { integer: \"-\" } units: \"") - + ((pynini.cross("/", "per") + delete_zero_or_one_space) | (pynini.accep("per") + pynutil.delete(" "))) - + pynutil.insert(NEMO_NON_BREAKING_SPACE) - + graph_unit - + pynutil.insert("\" preserve_order: true") - ) - - decimal_dash_alpha = ( - pynutil.insert("decimal { ") - + decimal.final_graph_wo_negative - + pynini.cross('-', '') - + pynutil.insert(" } units: \"") - + pynini.closure(SE_ALPHA, 1) - + pynutil.insert("\"") - ) - - decimal_times = ( - pynutil.insert("decimal { ") - + decimal.final_graph_wo_negative - + pynutil.insert(" } units: \"") - + (pynini.cross(pynini.union('x', "X"), 'x') | pynini.cross(pynini.union('x', "X"), ' geardde')) - + pynutil.insert("\"") - ) - - alpha_dash_decimal = ( - pynutil.insert("units: \"") - + pynini.closure(SE_ALPHA, 1) - + pynini.accep('-') - + pynutil.insert("\"") - + pynutil.insert(" decimal { ") - + decimal.final_graph_wo_negative - + pynutil.insert(" } preserve_order: true") - ) - - subgraph_fraction = ( - pynutil.insert("fraction { ") + fraction.graph + delete_space + pynutil.insert(" } ") + unit_plural - ) - - math_operations = pynini.string_file(get_abs_path("data/math_operations.tsv")) - delimiter = pynini.accep(" ") | pynutil.insert(" ") - - equals = pynini.cross("=", "lea") - - math = ( - (cardinal_graph | SE_ALPHA | greek) - + delimiter - + math_operations - + (delimiter | SE_ALPHA) - + cardinal_graph - + delimiter - + equals - + delimiter - + (cardinal_graph | SE_ALPHA | greek) - ) - - math = ( - pynutil.insert("units: \"math\" cardinal { integer: \"") - + math - + pynutil.insert("\" } preserve_order: true") - ) - final_graph = ( - subgraph_decimal - | subgraph_cardinal - | unit_graph - | decimal_dash_alpha - | decimal_times - | alpha_dash_decimal - | subgraph_fraction - | math - ) - - final_graph = self.add_tokens(final_graph) - self.fst = final_graph.optimize() - - def get_range(self, cardinal: GraphFst): - """ - Returns range forms for measure tagger, e.g. 2-3, 2x3, 2*2 - - Args: - cardinal: cardinal GraphFst - """ - range_graph = cardinal + pynini.cross(pynini.union("-", " - "), " till ") + cardinal - - for x in [" x ", "x"]: - range_graph |= cardinal + pynini.cross(x, " gånger ") + cardinal - - for x in ["*", " * "]: - range_graph |= cardinal + pynini.cross(x, " gånger ") + cardinal - return range_graph.optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/money.py b/nemo_text_processing/text_normalization/se/taggers/money.py index 64fc461bc..a12a74901 100644 --- a/nemo_text_processing/text_normalization/se/taggers/money.py +++ b/nemo_text_processing/text_normalization/se/taggers/money.py @@ -1,5 +1,5 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# Copyright (c) 2023, Jim O'Regan for Språkbanken Tal +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023, 2026, Jim O'Regan for Språkbanken Tal # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,219 +14,42 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import ( - NEMO_DIGIT, - NEMO_SIGMA, - GraphFst, - convert_space, - delete_space, - insert_space, -) -from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA, ensure_space -from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels from pynini.lib import pynutil -min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv")) -min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv")) -maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv"))) -maj_singular_nt = pynini.string_file((get_abs_path("data/money/currency_major_nt.tsv"))) -maj_plural = pynini.string_file((get_abs_path("data/money/currency_plurals.tsv"))) +from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, delete_zero_or_one_space +from nemo_text_processing.text_normalization.se.utils import get_abs_path class MoneyFst(GraphFst): - """ - Finite state transducer for classifying money, suppletive aware, e.g. - $12,05 -> money { integer_part: "tolv" currency_maj: "dollar" fractional_part: "fem" currency_min: "cent" preserve_order: true } - $12,0500 -> money { integer_part: "tolv" currency_maj: "dollar" fractional_part: "fem" currency_min: "cent" preserve_order: true } - $1 -> money { currency_maj: "dollar" integer_part: "en" } - $1,00 -> money { currency_maj: "dollar" integer_part: "en" } - $0,05 -> money { fractional_part: "fem" currency_min: "cent" preserve_order: true } - $1 miljon -> money { currency_maj: "dollar" integer_part: "en" quantity: "miljon" } - $1,2 miljon -> money { currency_maj: "dollar" integer_part: "en" fractional_part: "två" quantity: "miljon" } - $1,2320 -> money { currency_maj: "dollar" integer_part: "en" fractional_part: "two three two" } + """Classifies documented Northern Sámi integer currency expressions.""" - Args: - cardinal: CardinalFst - decimal: DecimalFst - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ - - def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + def __init__(self, cardinal: GraphFst, decimal=None, deterministic: bool = True): super().__init__(name="money", kind="classify", deterministic=deterministic) - cardinal_graph = cardinal.graph_no_one_en - cardinal_graph_ett = cardinal.graph_no_one - graph_decimal_final = decimal.final_graph_wo_negative_w_abbr_en - graph_decimal_final_ett = decimal.final_graph_wo_negative_w_abbr - - maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) - maj_singular_labels_nt = load_labels(get_abs_path("data/money/currency_major_nt.tsv")) - maj_unit_plural = convert_space(maj_singular @ maj_plural) - maj_unit_plural_nt = convert_space(maj_singular_nt @ maj_plural) - maj_unit_singular = convert_space(maj_singular) - maj_unit_singular_nt = convert_space(maj_singular_nt) - - self.maj_unit_plural = maj_unit_plural - self.maj_unit_plural_nt = maj_unit_plural_nt - self.maj_unit_singular = maj_unit_singular - self.maj_unit_singular_nt = maj_unit_singular_nt - - graph_maj_singular = pynutil.insert("currency_maj: \"") + maj_unit_singular + pynutil.insert("\"") - graph_maj_plural = pynutil.insert("currency_maj: \"") + maj_unit_plural + pynutil.insert("\"") - graph_maj_singular_nt = pynutil.insert("currency_maj: \"") + maj_unit_singular_nt + pynutil.insert("\"") - graph_maj_plural_nt = pynutil.insert("currency_maj: \"") + maj_unit_plural_nt + pynutil.insert("\"") - - optional_delete_fractional_zeros = pynini.closure( - pynutil.delete(",") + pynini.closure(pynutil.delete("0"), 1), 0, 1 - ) - - graph_integer_sg_en = pynutil.insert("integer_part: \"") + pynini.cross("1", "en") + pynutil.insert("\"") - graph_integer_sg_ett = pynutil.insert("integer_part: \"") + pynini.cross("1", "ett") + pynutil.insert("\"") - # only for decimals where third decimal after comma is non-zero or with quantity - decimal_delete_last_zeros = ( - pynini.closure(NEMO_DIGIT | pynutil.delete(" ")) - + pynini.accep(",") - + pynini.closure(NEMO_DIGIT, 2) - + (NEMO_DIGIT - "0") - + pynini.closure(pynutil.delete("0")) - ) - decimal_with_quantity = NEMO_SIGMA + SE_ALPHA - decimal_part = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final - decimal_part_ett = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final_ett - graph_decimal = pynini.union( - graph_maj_plural + ensure_space + decimal_part, - graph_maj_plural_nt + ensure_space + decimal_part_ett, - decimal_part_ett + ensure_space + graph_maj_plural_nt, - decimal_part + ensure_space + graph_maj_plural, + currency_nominative = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) + currency_genitive = pynini.string_file(get_abs_path("data/money/currency_major_gen.tsv")) + one = pynini.accep("1") @ cardinal.graph + non_one = pynini.difference(pynini.project(cardinal.graph, "input"), "1") @ cardinal.graph + separator = delete_zero_or_one_space + optional_zero_fraction = pynini.closure( + pynutil.delete(",") + (pynutil.delete("00") | pynutil.delete("–")), 0, 1 ) - graph_integer = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") - graph_integer_ett = pynutil.insert("integer_part: \"") + cardinal_graph_ett + pynutil.insert("\"") + def integer_token(graph): + return pynutil.insert('integer_part: "') + graph + pynutil.insert('" ') - graph_integer_only = graph_maj_singular + ensure_space + graph_integer_sg_en - graph_integer_only |= graph_maj_singular_nt + ensure_space + graph_integer_sg_ett - graph_integer_only |= graph_maj_plural + ensure_space + graph_integer - graph_integer_only |= graph_maj_plural_nt + ensure_space + graph_integer_ett - graph_integer_only |= graph_integer_sg_en + ensure_space + graph_maj_singular - graph_integer_only |= graph_integer_sg_ett + ensure_space + graph_maj_singular_nt - graph_integer_only |= graph_integer + ensure_space + graph_maj_plural - graph_integer_only |= graph_integer_ett + ensure_space + graph_maj_plural_nt - - final_graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_decimal - - # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits - # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10 - # not accepted: 002, 00, 0, - two_digits_fractional_part = ( - pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) - ) @ ( - (pynutil.delete("0") + (NEMO_DIGIT - "0")) - | ((NEMO_DIGIT - "0") + pynutil.insert("0")) - | ((NEMO_DIGIT - "0") + NEMO_DIGIT) - ) - - graph_min_singular = pynutil.insert(" currency_min: \"") + min_singular + pynutil.insert("\"") - graph_min_plural = pynutil.insert(" currency_min: \"") + min_plural + pynutil.insert("\"") - - maj_singular_labels_all = [(x[0], "okta") for x in maj_singular_labels] - # format ** dollars ** cent - decimal_graph_with_minor = None - integer_graph_reordered = None - decimal_default_reordered = None - for curr_symbol, one_form in maj_singular_labels_all: - preserve_order = pynutil.insert(" preserve_order: true") - if one_form == "en": - integer_plus_maj = graph_integer + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_plural) - integer_plus_maj |= ( - graph_integer_sg_en + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_singular) - ) - else: - integer_plus_maj = ( - graph_integer_ett + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_plural_nt) - ) - integer_plus_maj |= ( - graph_integer_sg_ett + insert_space + (pynutil.insert(curr_symbol) @ graph_maj_singular_nt) - ) - - integer_plus_maj_with_comma = pynini.compose( - NEMO_DIGIT - "0" + pynini.closure(NEMO_DIGIT | delete_space), integer_plus_maj - ) - integer_plus_maj = pynini.compose(pynini.closure(NEMO_DIGIT) - "0", integer_plus_maj) - integer_plus_maj |= integer_plus_maj_with_comma - - # all of the minor currency units are "en" - graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "en") - graph_fractional_one = pynutil.insert("fractional_part: \"") + graph_fractional_one + pynutil.insert("\"") - graph_fractional = ( - two_digits_fractional_part - @ (pynini.closure(NEMO_DIGIT, 1, 2) - "1") - @ cardinal.graph_hundreds_component_at_least_one_non_zero_digit_en - ) - graph_fractional = pynutil.insert("fractional_part: \"") + graph_fractional + pynutil.insert("\"") - - fractional_plus_min = graph_fractional + ensure_space + (pynutil.insert(curr_symbol) @ graph_min_plural) - fractional_plus_min |= ( - graph_fractional_one + ensure_space + (pynutil.insert(curr_symbol) @ graph_min_singular) - ) - - decimal_graph_with_minor_curr = integer_plus_maj + pynini.cross(",", " ") + fractional_plus_min - - if not deterministic: - decimal_graph_with_minor_curr |= pynutil.add_weight( - integer_plus_maj - + pynini.cross(",", " ") - + pynutil.insert("fractional_part: \"") - + two_digits_fractional_part @ cardinal.graph_hundreds_component_at_least_one_non_zero_digit_en - + pynutil.insert("\""), - weight=0.0001, - ) - default_fraction_graph = (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final - - decimal_graph_with_minor_curr |= ( - pynini.closure(pynutil.delete("0"), 0, 1) + pynutil.delete(",") + fractional_plus_min - ) - decimal_graph_with_minor_curr = ( - pynutil.delete(curr_symbol) + decimal_graph_with_minor_curr + preserve_order - ) - - decimal_graph_with_minor = ( - decimal_graph_with_minor_curr - if decimal_graph_with_minor is None - else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr).optimize() - ) - - if not deterministic: - integer_graph_reordered_curr = ( - pynutil.delete(curr_symbol) + integer_plus_maj + preserve_order - ).optimize() - - integer_graph_reordered = ( - integer_graph_reordered_curr - if integer_graph_reordered is None - else pynini.union(integer_graph_reordered, integer_graph_reordered_curr).optimize() - ) - decimal_default_reordered_curr = ( - pynutil.delete(curr_symbol) - + default_fraction_graph - + ensure_space - + pynutil.insert(curr_symbol) @ graph_maj_plural - ) - - decimal_default_reordered = ( - decimal_default_reordered_curr - if decimal_default_reordered is None - else pynini.union(decimal_default_reordered, decimal_default_reordered_curr) - ).optimize() - - # weight for SH - final_graph |= pynutil.add_weight(decimal_graph_with_minor, -0.0001) + def currency_token(graph): + return pynutil.insert('currency_maj: "') + graph + pynutil.insert('"') + singular = integer_token(one) + optional_zero_fraction + separator + currency_token(currency_nominative) + governed = integer_token(non_one) + optional_zero_fraction + separator + currency_token(currency_genitive) if not deterministic: - final_graph |= integer_graph_reordered | decimal_default_reordered - # to handle "$2.00" cases - final_graph |= pynini.compose( - NEMO_SIGMA + pynutil.delete(",") + pynini.closure(pynutil.delete("0"), 1), integer_graph_reordered + governed |= ( + integer_token(non_one) + optional_zero_fraction + separator + currency_token(currency_nominative) ) - final_graph = self.add_tokens(final_graph.optimize()) - self.fst = final_graph.optimize() + + self.fst = self.add_tokens(singular | governed).optimize() + self.graph = ( + one + optional_zero_fraction + separator + pynutil.insert(" ") + currency_nominative + | non_one + optional_zero_fraction + separator + pynutil.insert(" ") + currency_genitive + ).optimize() diff --git a/nemo_text_processing/text_normalization/se/taggers/ordinal.py b/nemo_text_processing/text_normalization/se/taggers/ordinal.py index bbbcfe7c2..708667ac7 100644 --- a/nemo_text_processing/text_normalization/se/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/se/taggers/ordinal.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -23,8 +25,7 @@ insert_space, ) from nemo_text_processing.text_normalization.se.graph_utils import SE_ALPHA -from nemo_text_processing.text_normalization.se.utils import get_abs_path -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) @@ -42,7 +43,7 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': Returns: fst: A pynini.FstLike object """ - exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string cardinal_separator = pynini.union(NEMO_SPACE, ".") @@ -172,7 +173,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): self.graph = ( ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) @@ -186,6 +187,28 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): self.graph_bare_ordinals = filter_punctuation(self.graph).optimize() self.graph = (self.graph_bare_ordinals + pynutil.delete(".")).optimize() + # Riektačállinrávvagat (Sámediggi, revised 2019), p. 61: + # "Ortnetloguide (ortnetadjektiivvaide) maid loahppaoassin lea -logát + # gustojit čuovvovaš rávvagat." + logat = pynini.closure(NEMO_SIGMA) + pynini.accep("logát") + logat_ordinal = (self.graph_bare_ordinals @ logat).optimize() + logad_stem = logat_ordinal @ pynini.cdrewrite(pynutil.delete("t"), "", "[EOS]", NEMO_SIGMA) + + self.graphs = {"nom_sg": self.graph_bare_ordinals} + ordinal_case_suffixes = load_labels(get_abs_path("data/inflection/ordinal_case_suffixes.tsv")) + explicit_case_graphs = [] + for case, suffix in ordinal_case_suffixes: + case_graph = (logad_stem + pynutil.insert(suffix)).optimize() + self.graphs[case] = case_graph + explicit_case_graphs.append(case_graph + pynutil.delete(":") + pynutil.delete(suffix)) + self.graph_dict = self.graphs + self.graph_suffixed = pynini.union(*explicit_case_graphs).optimize() + + self.graph |= self.graph_suffixed + if not deterministic: + self.graph |= pynini.union(*self.graphs.values()) + self.graph = self.graph.optimize() + final_graph = pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") final_graph = self.add_tokens(final_graph) diff --git a/nemo_text_processing/text_normalization/se/taggers/telephone.py b/nemo_text_processing/text_normalization/se/taggers/telephone.py index 6b82d36cd..7c37217db 100644 --- a/nemo_text_processing/text_normalization/se/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/se/taggers/telephone.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_SPACE, GraphFst, @@ -24,7 +26,6 @@ from nemo_text_processing.text_normalization.se.graph_utils import ensure_space from nemo_text_processing.text_normalization.se.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.se.utils import get_abs_path -from pynini.lib import pynutil class TelephoneFst(GraphFst): @@ -48,8 +49,8 @@ class TelephoneFst(GraphFst): https://codegolf.stackexchange.com/questions/195787/format-a-swedish-phone-number Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) """ def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/se/taggers/time.py b/nemo_text_processing/text_normalization/se/taggers/time.py index 9a7ef910a..181c8f609 100644 --- a/nemo_text_processing/text_normalization/se/taggers/time.py +++ b/nemo_text_processing/text_normalization/se/taggers/time.py @@ -15,6 +15,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -26,7 +28,6 @@ ) from nemo_text_processing.text_normalization.se.graph_utils import ensure_space from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels -from pynini.lib import pynutil QUARTERS = {15: "kvárta badjel", 30: "beal", 45: "kvárta váile"} @@ -133,7 +134,11 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): pynutil.insert("seconds: \"") + pynini.cross("00", "nolla nolla") + pynutil.insert("\"") ) final_time_zone = pynutil.insert("zone: \"") + convert_space(time_zone_graph) + pynutil.insert("\"") - final_time_zone_optional = pynini.closure(NEMO_SPACE + final_time_zone, 0, 1,) + final_time_zone_optional = pynini.closure( + NEMO_SPACE + final_time_zone, + 0, + 1, + ) # 2:30 pm, 02:30, 2:00 graph_hm_kl = ( diff --git a/nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py index 6dad08718..c49087aeb 100644 --- a/nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/se/taggers/tokenize_and_classify.py @@ -17,6 +17,8 @@ import time import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_WHITE_SPACE, GraphFst, @@ -34,11 +36,9 @@ from nemo_text_processing.text_normalization.se.taggers.measure import MeasureFst from nemo_text_processing.text_normalization.se.taggers.money import MoneyFst from nemo_text_processing.text_normalization.se.taggers.ordinal import OrdinalFst -from nemo_text_processing.text_normalization.se.taggers.telephone import TelephoneFst from nemo_text_processing.text_normalization.se.taggers.time import TimeFst from nemo_text_processing.text_normalization.se.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.se.taggers.word import WordFst -from pynini.lib import pynutil class ClassifyFst(GraphFst): @@ -99,10 +99,9 @@ def __init__( # fraction_graph = fraction.fst # logging.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") - # start_time = time.time() - # measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) - # measure_graph = measure.fst - # logging.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") + start_time = time.time() + measure_graph = MeasureFst(cardinal=cardinal, deterministic=deterministic).fst + logging.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") start_time = time.time() date_graph = DateFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic).fst @@ -112,17 +111,13 @@ def __init__( time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst logging.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") - start_time = time.time() - telephone_graph = TelephoneFst(deterministic=deterministic).fst - logging.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") - start_time = time.time() electonic_graph = ElectronicFst(deterministic=deterministic).fst logging.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") - # start_time = time.time() - # money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst - # logging.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") + start_time = time.time() + money_graph = MoneyFst(cardinal=cardinal, deterministic=deterministic).fst + logging.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") start_time = time.time() whitelist_graph = WhiteListFst( @@ -144,10 +139,10 @@ def __init__( | pynutil.add_weight(time_graph, 1.1) | pynutil.add_weight(date_graph, 1.09) # | pynutil.add_weight(decimal_graph, 1.1) - # | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) | pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(ordinal_graph, 1.1) - # | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(money_graph, 1.1) # | pynutil.add_weight(telephone_graph, 1.1) | pynutil.add_weight(electonic_graph, 1.1) # | pynutil.add_weight(fraction_graph, 1.1) diff --git a/nemo_text_processing/text_normalization/se/taggers/whitelist.py b/nemo_text_processing/text_normalization/se/taggers/whitelist.py index fe74701d6..2a10c88c5 100644 --- a/nemo_text_processing/text_normalization/se/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/se/taggers/whitelist.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.se.utils import get_abs_path, load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/se/taggers/word.py b/nemo_text_processing/text_normalization/se/taggers/word.py index 98e211c19..74ad8a3a3 100644 --- a/nemo_text_processing/text_normalization/se/taggers/word.py +++ b/nemo_text_processing/text_normalization/se/taggers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/se/utils.py b/nemo_text_processing/text_normalization/se/utils.py index 91262904d..c361fd832 100644 --- a/nemo_text_processing/text_normalization/se/utils.py +++ b/nemo_text_processing/text_normalization/se/utils.py @@ -20,6 +20,28 @@ CASE_KEYS = ["ess", "com_pl", "com_sg", "gen_sg", "gen_pl", "ill_pl", "ill_sg", "loc_sg", "nom_pl"] CASE_KEYS_EXT = CASE_KEYS + ["nom_sg"] +CASE_ALIASES = { + "acc_sg": "nom_sg", + "acc_pl": "gen_pl", + "loc_pl": "com_sg", +} + +CARDINAL_CASE_KEYS = [ + "nom_sg", + "nom_pl", + "acc_sg", + "acc_pl", + "gen_sg", + "gen_pl", + "ill_sg", + "ill_pl", + "loc_sg", + "loc_pl", + "com_sg", + "com_pl", + "ess", +] + def get_abs_path(rel_path): """ @@ -27,7 +49,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path diff --git a/nemo_text_processing/text_normalization/se/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/se/verbalizers/cardinal.py index bd0551a15..3bc21a0b5 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/se/verbalizers/date.py b/nemo_text_processing/text_normalization/se/verbalizers/date.py index cd20405be..3632294af 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/date.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,7 +22,6 @@ delete_extra_space, delete_space, ) -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/se/verbalizers/decimals.py b/nemo_text_processing/text_normalization/se/verbalizers/decimals.py index b07a51d57..dd71814a1 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/decimals.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/decimals.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -19,19 +21,18 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. - decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones - decimal { integer_part: "un" quantity: "billón" } -> un billón + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones + decimal { integer_part: "un" quantity: "billón" } -> un billón Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="decimal", kind="classify", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/se/verbalizers/electronic.py b/nemo_text_processing/text_normalization/se/verbalizers/electronic.py index 5ff4a97a9..10e0fcac1 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/electronic.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -20,7 +22,6 @@ insert_space, ) from nemo_text_processing.text_normalization.se.utils import get_abs_path -from pynini.lib import pynutil class ElectronicFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/se/verbalizers/fraction.py b/nemo_text_processing/text_normalization/se/verbalizers/fraction.py index 27357ecb1..003ce2e97 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/fraction.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/se/verbalizers/measure.py b/nemo_text_processing/text_normalization/se/verbalizers/measure.py index 4ca928008..447150a7a 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + class MeasureFst(GraphFst): """ @@ -23,7 +24,7 @@ class MeasureFst(GraphFst): measure { negative: "true" cardinal { integer: "twelve" } units: "kilograms" } -> minus twelve kilograms measure { decimal { integer_part: "twelve" fractional_part: "five" } units: "kilograms" } -> twelve point five kilograms tokens { measure { units: "covid" decimal { integer_part: "nineteen" fractional_part: "five" } } } -> covid nineteen point five - + Args: decimal: DecimalFst cardinal: CardinalFst diff --git a/nemo_text_processing/text_normalization/se/verbalizers/money.py b/nemo_text_processing/text_normalization/se/verbalizers/money.py index 6ffa83cd2..115edd44c 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/money.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_preserve_order, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/se/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/se/verbalizers/ordinal.py index 99e1592e5..df16e2562 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/ordinal.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/se/verbalizers/telephone.py b/nemo_text_processing/text_normalization/se/verbalizers/telephone.py index 7aa5bf334..3042f47e5 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/telephone.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/telephone.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SPACE, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TelephoneFst(GraphFst): @@ -39,7 +40,11 @@ def __init__(self, deterministic: bool = True): country_code = pynutil.delete("country_code: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") - optional_country_code = pynini.closure(country_code + delete_space + insert_space, 0, 1,) + optional_country_code = pynini.closure( + country_code + delete_space + insert_space, + 0, + 1, + ) prompt_part = ( pynutil.delete("prompt: \"") diff --git a/nemo_text_processing/text_normalization/se/verbalizers/time.py b/nemo_text_processing/text_normalization/se/verbalizers/time.py index 93e0434b1..7d7c5525b 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -22,7 +24,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py index d0f4d82a6..33b624c8f 100644 --- a/nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/se/verbalizers/verbalize_final.py @@ -16,6 +16,8 @@ import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.se.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil class VerbalizeFinalFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/graph_utils.py b/nemo_text_processing/text_normalization/sv/graph_utils.py index 001addc7c..b229ca2fa 100644 --- a/nemo_text_processing/text_normalization/sv/graph_utils.py +++ b/nemo_text_processing/text_normalization/sv/graph_utils.py @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space from pynini.lib import byte, pynutil +from nemo_text_processing.text_normalization.en.graph_utils import delete_space, insert_space + from .utils import get_abs_path, load_labels _ALPHA_UPPER = "ABCDEFGHIJKLMNOPQRSTUVWXYZÅÄÖÜÉ" diff --git a/nemo_text_processing/text_normalization/sv/taggers/abbreviation.py b/nemo_text_processing/text_normalization/sv/taggers/abbreviation.py index 5aac4cd13..a5a5b8ede 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/abbreviation.py +++ b/nemo_text_processing/text_normalization/sv/taggers/abbreviation.py @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space from nemo_text_processing.text_normalization.sv.graph_utils import SV_UPPER -from pynini.lib import pynutil class AbbreviationFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/taggers/cardinal.py b/nemo_text_processing/text_normalization/sv/taggers/cardinal.py index 7f06fcd5d..750ff867b 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/sv/taggers/cardinal.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.sv.graph_utils import SV_ALPHA from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil def make_million(number: str, non_zero_no_one: 'pynini.FstLike', deterministic: bool = True) -> 'pynini.FstLike': @@ -68,7 +69,7 @@ def filter_punctuation(fst: 'pynini.FstLike') -> 'pynini.FstLike': Returns: fst: A pynini.FstLike object """ - exactly_three_digits = NEMO_DIGIT ** 3 # for blocks of three + exactly_three_digits = NEMO_DIGIT**3 # for blocks of three up_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) # for start of string cardinal_separator = NEMO_SPACE @@ -248,7 +249,7 @@ def __init__(self, deterministic: bool = True): self.graph = ( ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) @@ -275,30 +276,27 @@ def __init__(self, deterministic: bool = True): zero_space = zero + insert_space self.zero_space = zero_space self.three_digits_read = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + zero_space + ((NEMO_DIGIT**2) @ graph_tens), zero_space + zero_space + digit, ) self.three_digits_read_en = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, - zero_space + ((NEMO_DIGIT ** 2) @ graph_tens), + zero_space + ((NEMO_DIGIT**2) @ graph_tens), zero_space + zero_space + digit, ) self.three_digits_read_frac = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, zero_space + digit + insert_space + digit, ) self.three_digits_read_frac_en = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, zero_space + digit + insert_space + digit, ) self.two_or_three_digits_read_frac = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, zero_space + single_digits_graph + pynini.closure(insert_space + digit, 0, 1), single_digits_graph + pynini.closure(insert_space + single_digits_graph, 3), @@ -306,7 +304,7 @@ def __init__(self, deterministic: bool = True): single_digits_graph, ) self.two_or_three_digits_read_frac_en = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), zero_space + single_digits_graph + pynini.closure(insert_space + single_digits_graph, 0, 1), @@ -315,9 +313,8 @@ def __init__(self, deterministic: bool = True): single_digits_graph, ) self.two_or_three_digits_read_frac_both = pynini.union( - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) - @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, - ((NEMO_DIGIT - "0") + (NEMO_DIGIT ** 2)) + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one, + ((NEMO_DIGIT - "0") + (NEMO_DIGIT**2)) @ self.graph_hundreds_component_at_least_one_non_zero_digit_no_one_en, ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ graph_tens, ((NEMO_DIGIT - "0") + NEMO_DIGIT) @ (graph_tens @ pynini.cdrewrite(ett_to_en, "", "[EOS]", NEMO_SIGMA)), diff --git a/nemo_text_processing/text_normalization/sv/taggers/date.py b/nemo_text_processing/text_normalization/sv/taggers/date.py index ce2bc8f74..58e3d0d3e 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/date.py +++ b/nemo_text_processing/text_normalization/sv/taggers/date.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SPACE, @@ -22,7 +24,6 @@ ) from nemo_text_processing.text_normalization.sv.graph_utils import SV_ALPHA from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil class DateFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/taggers/decimal.py b/nemo_text_processing/text_normalization/sv/taggers/decimal.py index a63b939fb..2ac0cecc1 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/sv/taggers/decimal.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil def get_quantity( diff --git a/nemo_text_processing/text_normalization/sv/taggers/electronic.py b/nemo_text_processing/text_normalization/sv/taggers/electronic.py index f0896edf8..75a24b205 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/electronic.py +++ b/nemo_text_processing/text_normalization/sv/taggers/electronic.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_ALPHA, NEMO_DIGIT, GraphFst, insert_space from nemo_text_processing.text_normalization.sv.utils import get_abs_path, load_labels -from pynini.lib import pynutil class ElectronicFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/taggers/fraction.py b/nemo_text_processing/text_normalization/sv/taggers/fraction.py index a53d723b8..69ab07447 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/sv/taggers/fraction.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.sv.graph_utils import ensure_space from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil class FractionFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/taggers/measure.py b/nemo_text_processing/text_normalization/sv/taggers/measure.py index 83dcf9b0a..4da3f81c2 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/measure.py +++ b/nemo_text_processing/text_normalization/sv/taggers/measure.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NON_BREAKING_SPACE, NEMO_SIGMA, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.sv.graph_utils import SV_ALPHA, TO_LOWER from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil class MeasureFst(GraphFst): @@ -80,7 +81,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, de ) optional_graph_unit2 = pynini.closure( - delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, 0, 1, + delete_zero_or_one_space + pynutil.insert(NEMO_NON_BREAKING_SPACE) + graph_unit2, + 0, + 1, ) unit_plural = ( diff --git a/nemo_text_processing/text_normalization/sv/taggers/money.py b/nemo_text_processing/text_normalization/sv/taggers/money.py index 37bd8f9e4..16858cc5c 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/money.py +++ b/nemo_text_processing/text_normalization/sv/taggers/money.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -24,7 +26,6 @@ ) from nemo_text_processing.text_normalization.sv.graph_utils import SV_ALPHA, ensure_space from nemo_text_processing.text_normalization.sv.utils import get_abs_path, load_labels -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/taggers/ordinal.py b/nemo_text_processing/text_normalization/sv/taggers/ordinal.py index 59dbe528f..0877ca08f 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/sv/taggers/ordinal.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -25,13 +27,12 @@ ) from nemo_text_processing.text_normalization.sv.taggers.cardinal import filter_punctuation, make_million from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil class OrdinalFst(GraphFst): """ Finite state transducer for classifying ordinal - "21:a" -> ordinal { integer: "tjugoförsta" } + "21:a" -> ordinal { integer: "tjugoförsta" } Args: cardinal: CardinalFst deterministic: if True will provide a single transduction option, @@ -94,7 +95,10 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): hundreds |= pynini.cross("1", "ett hundra") hundreds |= digit + pynutil.insert(NEMO_SPACE) + pynutil.insert("hundra") - graph_hundreds = hundreds + pynini.union(graph_tens, (pynutil.delete("0") + graph_digit),) + graph_hundreds = hundreds + pynini.union( + graph_tens, + (pynutil.delete("0") + graph_digit), + ) if not deterministic: graph_hundreds |= hundreds + pynini.union( (graph_teens | pynutil.insert(NEMO_SPACE) + graph_teens), (pynini.cross("0", NEMO_SPACE) + graph_digit) @@ -178,7 +182,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): self.graph = ( ((NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0)) @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", NEMO_SIGMA) - @ NEMO_DIGIT ** 24 + @ NEMO_DIGIT**24 @ graph @ pynini.cdrewrite(delete_space, "[BOS]", "", NEMO_SIGMA) @ pynini.cdrewrite(delete_space, "", "[EOS]", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/sv/taggers/telephone.py b/nemo_text_processing/text_normalization/sv/taggers/telephone.py index 7d85072ef..a03e0430b 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/sv/taggers/telephone.py @@ -14,6 +14,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SPACE, @@ -25,7 +27,6 @@ from nemo_text_processing.text_normalization.sv.graph_utils import ensure_space from nemo_text_processing.text_normalization.sv.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil class TelephoneFst(GraphFst): @@ -49,8 +50,8 @@ class TelephoneFst(GraphFst): https://codegolf.stackexchange.com/questions/195787/format-a-swedish-phone-number Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) """ def __init__(self, deterministic: bool = True): diff --git a/nemo_text_processing/text_normalization/sv/taggers/time.py b/nemo_text_processing/text_normalization/sv/taggers/time.py index d89182bc4..cb5067058 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/time.py +++ b/nemo_text_processing/text_normalization/sv/taggers/time.py @@ -15,6 +15,8 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_DIGIT, NEMO_SIGMA, @@ -26,7 +28,6 @@ ) from nemo_text_processing.text_normalization.sv.graph_utils import ensure_space from nemo_text_processing.text_normalization.sv.utils import get_abs_path, load_labels -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -105,7 +106,11 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): final_suffix = pynutil.insert("suffix: \"") + convert_space(suffix_graph) + pynutil.insert("\"") final_suffix_optional = pynini.closure(ensure_space + final_suffix, 0, 1) final_time_zone = pynutil.insert("zone: \"") + convert_space(time_zone_graph) + pynutil.insert("\"") - final_time_zone_optional = pynini.closure(NEMO_SPACE + final_time_zone, 0, 1,) + final_time_zone_optional = pynini.closure( + NEMO_SPACE + final_time_zone, + 0, + 1, + ) # 2:30 pm, 02:30, 2:00 graph_hm_kl = ( diff --git a/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify.py index 1a9637c91..ff2495524 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import time import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_WHITE_SPACE, GraphFst, @@ -38,7 +39,7 @@ from nemo_text_processing.text_normalization.sv.taggers.time import TimeFst from nemo_text_processing.text_normalization.sv.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.sv.taggers.word import WordFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -75,69 +76,69 @@ def __init__( ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f'ClassifyFst.fst was restored from {far_file}.') + logger.info(f'ClassifyFst.fst was restored from {far_file}.') else: - logging.info(f"Creating ClassifyFst grammars.") + logger.info(f"Creating ClassifyFst grammars.") start_time = time.time() cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = cardinal.fst - logging.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") + logger.debug(f"cardinal: {time.time() - start_time: .2f}s -- {cardinal_graph.num_states()} nodes") start_time = time.time() ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) ordinal_graph = ordinal.fst - logging.debug(f"ordinal: {time.time() - start_time: .2f}s -- {ordinal_graph.num_states()} nodes") + logger.debug(f"ordinal: {time.time() - start_time: .2f}s -- {ordinal_graph.num_states()} nodes") start_time = time.time() decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) decimal_graph = decimal.fst - logging.debug(f"decimal: {time.time() - start_time: .2f}s -- {decimal_graph.num_states()} nodes") + logger.debug(f"decimal: {time.time() - start_time: .2f}s -- {decimal_graph.num_states()} nodes") start_time = time.time() fraction = FractionFst(deterministic=deterministic, ordinal=ordinal, cardinal=cardinal) fraction_graph = fraction.fst - logging.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") + logger.debug(f"fraction: {time.time() - start_time: .2f}s -- {fraction_graph.num_states()} nodes") start_time = time.time() measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) measure_graph = measure.fst - logging.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") + logger.debug(f"measure: {time.time() - start_time: .2f}s -- {measure_graph.num_states()} nodes") start_time = time.time() date_graph = DateFst(cardinal=cardinal, ordinal=ordinal, deterministic=deterministic).fst - logging.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes") + logger.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes") start_time = time.time() time_graph = TimeFst(cardinal=cardinal, deterministic=deterministic).fst - logging.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") + logger.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes") start_time = time.time() telephone_graph = TelephoneFst(deterministic=deterministic).fst - logging.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") + logger.debug(f"telephone: {time.time() - start_time: .2f}s -- {telephone_graph.num_states()} nodes") start_time = time.time() electonic_graph = ElectronicFst(deterministic=deterministic).fst - logging.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") + logger.debug(f"electronic: {time.time() - start_time: .2f}s -- {electonic_graph.num_states()} nodes") start_time = time.time() money_graph = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic).fst - logging.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") + logger.debug(f"money: {time.time() - start_time: .2f}s -- {money_graph.num_states()} nodes") start_time = time.time() whitelist_graph = WhiteListFst( input_case=input_case, deterministic=deterministic, input_file=whitelist ).fst - logging.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") + logger.debug(f"whitelist: {time.time() - start_time: .2f}s -- {whitelist_graph.num_states()} nodes") start_time = time.time() punctuation = PunctuationFst(deterministic=deterministic) punct_graph = punctuation.fst - logging.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") + logger.debug(f"punct: {time.time() - start_time: .2f}s -- {punct_graph.num_states()} nodes") start_time = time.time() word_graph = WordFst(deterministic=deterministic).fst - logging.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") + logger.debug(f"word: {time.time() - start_time: .2f}s -- {word_graph.num_states()} nodes") classify = ( pynutil.add_weight(whitelist_graph, 1.01) @@ -185,4 +186,3 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify_with_audio.py b/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify_with_audio.py index ab826476e..4470a4a73 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify_with_audio.py +++ b/nemo_text_processing/text_normalization/sv/taggers/tokenize_and_classify_with_audio.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_CHAR, NEMO_DIGIT, @@ -52,7 +53,7 @@ from nemo_text_processing.text_normalization.sv.verbalizers.ordinal import OrdinalFst as vOrdinalFst from nemo_text_processing.text_normalization.sv.verbalizers.telephone import TelephoneFst as vTelephoneFst from nemo_text_processing.text_normalization.sv.verbalizers.time import TimeFst as vTimeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class ClassifyFst(GraphFst): @@ -89,7 +90,7 @@ def __init__( ) if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - logging.info(f'ClassifyFst.fst was restored from {far_file}.') + logger.info(f'ClassifyFst.fst was restored from {far_file}.') else: cardinal = CardinalFst(deterministic=deterministic) cardinal_graph = cardinal.fst @@ -194,7 +195,6 @@ def __init__( if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f'ClassifyFst grammars are saved to {far_file}.') # to remove normalization options that still contain digits and some special symbols # e.g., "P&E" -> {P and E, P&E}, "P & E" will be removed from the list of normalization options diff --git a/nemo_text_processing/text_normalization/sv/taggers/whitelist.py b/nemo_text_processing/text_normalization/sv/taggers/whitelist.py index 1c3569b59..e8bf8151c 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/sv/taggers/whitelist.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.sv.utils import get_abs_path, load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/taggers/word.py b/nemo_text_processing/text_normalization/sv/taggers/word.py index 98e211c19..74ad8a3a3 100644 --- a/nemo_text_processing/text_normalization/sv/taggers/word.py +++ b/nemo_text_processing/text_normalization/sv/taggers/word.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, GraphFst + class WordFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/sv/verbalizers/cardinal.py index 3dec2cfa1..9412c4600 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst + class CardinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/date.py b/nemo_text_processing/text_normalization/sv/verbalizers/date.py index 7f45297e6..81ef6658a 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/date.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SPACE, @@ -19,7 +21,6 @@ delete_preserve_order, ) from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil era_words = pynini.string_file(get_abs_path("data/dates/era_words.tsv")) diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/decimals.py b/nemo_text_processing/text_normalization/sv/verbalizers/decimals.py index b07a51d57..dd71814a1 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/decimals.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/decimals.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, @@ -19,19 +21,18 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class DecimalFst(GraphFst): """ - Finite state transducer for classifying decimal, e.g. - decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones - decimal { integer_part: "un" quantity: "billón" } -> un billón + Finite state transducer for classifying decimal, e.g. + decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones + decimal { integer_part: "un" quantity: "billón" } -> un billón Args: - deterministic: if True will provide a single transduction option, - for False multiple transduction are generated (used for audio-based normalization) - """ + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ def __init__(self, deterministic: bool = True): super().__init__(name="decimal", kind="classify", deterministic=deterministic) diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/electronic.py b/nemo_text_processing/text_normalization/sv/verbalizers/electronic.py index 562f0c884..0cbbeb793 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/electronic.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/electronic.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -21,7 +23,6 @@ ) from nemo_text_processing.text_normalization.sv.graph_utils import bos_or_space, eos_or_space from nemo_text_processing.text_normalization.sv.utils import get_abs_path -from pynini.lib import pynutil digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))) zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv"))) diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/fraction.py b/nemo_text_processing/text_normalization/sv/verbalizers/fraction.py index 27357ecb1..003ce2e97 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/fraction.py @@ -14,9 +14,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, insert_space + class FractionFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/measure.py b/nemo_text_processing/text_normalization/sv/verbalizers/measure.py index 0af155d2b..447150a7a 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + class MeasureFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/money.py b/nemo_text_processing/text_normalization/sv/verbalizers/money.py index 30a33ceae..8610dc26b 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/money.py @@ -13,13 +13,14 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, GraphFst, delete_extra_space, delete_preserve_order, ) -from pynini.lib import pynutil class MoneyFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/sv/verbalizers/ordinal.py index 8a7217b15..ef46cf682 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/ordinal.py @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): """ diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/telephone.py b/nemo_text_processing/text_normalization/sv/verbalizers/telephone.py index 2eb79a759..6656e3445 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/telephone.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/telephone.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SPACE, @@ -20,7 +22,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TelephoneFst(GraphFst): @@ -39,7 +40,11 @@ def __init__(self, deterministic: bool = True): country_code = pynutil.delete("country_code: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") - optional_country_code = pynini.closure(country_code + delete_space + insert_space, 0, 1,) + optional_country_code = pynini.closure( + country_code + delete_space + insert_space, + 0, + 1, + ) number_part = ( pynutil.delete("number_part: \"") diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/time.py b/nemo_text_processing/text_normalization/sv/verbalizers/time.py index ae4c7b828..c715b3545 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/time.py @@ -13,6 +13,8 @@ # limitations under the License. import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( NEMO_NOT_QUOTE, NEMO_SIGMA, @@ -22,7 +24,6 @@ delete_space, insert_space, ) -from pynini.lib import pynutil class TimeFst(GraphFst): diff --git a/nemo_text_processing/text_normalization/sv/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/sv/verbalizers/verbalize_final.py index 647bfe6b6..52cf8de07 100644 --- a/nemo_text_processing/text_normalization/sv/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/sv/verbalizers/verbalize_final.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.en.graph_utils import ( GraphFst, delete_extra_space, @@ -24,7 +25,7 @@ ) from nemo_text_processing.text_normalization.en.verbalizers.word import WordFst from nemo_text_processing.text_normalization.sv.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil +from nemo_text_processing.utils.logging import logger class VerbalizeFinalFst(GraphFst): @@ -47,7 +48,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ far_file = os.path.join(cache_dir, f"sv_tn_{deterministic}_deterministic_verbalizer.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["verbalize"] - logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') else: verbalize = VerbalizeFst(deterministic=deterministic).fst word = WordFst(deterministic=deterministic).fst @@ -66,4 +67,4 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = graph.optimize() if far_file: generator_main(far_file, {"verbalize": self.fst}) - logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") + logger.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/token_parser.py b/nemo_text_processing/text_normalization/token_parser.py index 638b71bbf..4adcd7d7f 100644 --- a/nemo_text_processing/text_normalization/token_parser.py +++ b/nemo_text_processing/text_normalization/token_parser.py @@ -34,7 +34,7 @@ def __call__(self, text): Args: text: text to be parsed - + """ self.text = text self.len_text = len(text) @@ -107,11 +107,11 @@ def parse_token_value(self) -> Union[str, dict]: def parse_char(self, exp) -> bool: """ - Parses character + Parses character Args: exp: character to read in - + Returns true if successful """ assert self.char == exp @@ -124,7 +124,7 @@ def parse_chars(self, exp) -> bool: Args: exp: characters to read in - + Returns true if successful """ ok = False @@ -181,8 +181,8 @@ def parse_ws(self): def read(self): """ - Reads in next char. - + Reads in next char. + Returns true if not EOS """ if self.index < self.len_text - 1: # should be unique diff --git a/nemo_text_processing/text_normalization/utils_audio_based.py b/nemo_text_processing/text_normalization/utils_audio_based.py index 0f7e05865..2e9626d9e 100644 --- a/nemo_text_processing/text_normalization/utils_audio_based.py +++ b/nemo_text_processing/text_normalization/utils_audio_based.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from typing import Dict from cdifflib import CSequenceMatcher +from nemo_text_processing.utils.logging import logger + MATCH = "match" NONMATCH = "non-match" SEMIOTIC_TAG = "[SEMIOTIC_SPAN]" @@ -181,9 +182,9 @@ def adjust_boundaries(norm_raw_diffs: Dict, norm_pred_diffs: Dict, raw: str, nor cur_semiotic = " ".join(raw_list[item[1][0] : item[1][1]]) cur_pred_text = " ".join(pred_text_list[item[2][0] : item[2][1]]) cur_norm_span = " ".join(norm_list[item[0][0] : item[0][1]]) - logging.debug(f"cur_semiotic: {cur_semiotic}") - logging.debug(f"cur_pred_text: {cur_pred_text}") - logging.debug(f"cur_norm_span: {cur_norm_span}") + logger.debug(f"cur_semiotic: {cur_semiotic}") + logger.debug(f"cur_pred_text: {cur_pred_text}") + logger.debug(f"cur_norm_span: {cur_norm_span}") # if cur_pred_text is an empty string if item[2][0] == item[2][1]: diff --git a/nemo_text_processing/text_normalization/vi/__init__.py b/nemo_text_processing/text_normalization/vi/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/__init__.py b/nemo_text_processing/text_normalization/vi/data/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/date/__init__.py b/nemo_text_processing/text_normalization/vi/data/date/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/date/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/date/days.tsv b/nemo_text_processing/text_normalization/vi/data/date/days.tsv new file mode 100644 index 000000000..5b70479a6 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/date/days.tsv @@ -0,0 +1,40 @@ +01 một +02 hai +03 ba +04 bốn +05 năm +06 sáu +07 bảy +08 tám +09 chín +1 một +2 hai +3 ba +4 bốn +5 năm +6 sáu +7 bảy +8 tám +9 chín +10 mười +11 mười một +12 mười hai +13 mười ba +14 mười bốn +15 mười lăm +16 mười sáu +17 mười bảy +18 mười tám +19 mười chín +20 hai mươi +21 hai mươi mốt +22 hai mươi hai +23 hai mươi ba +24 hai mươi bốn +25 hai mươi lăm +26 hai mươi sáu +27 hai mươi bảy +28 hai mươi tám +29 hai mươi chín +30 ba mươi +31 ba mươi mốt \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/date/months.tsv b/nemo_text_processing/text_normalization/vi/data/date/months.tsv new file mode 100644 index 000000000..fb836fba1 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/date/months.tsv @@ -0,0 +1,21 @@ +1 một +2 hai +3 ba +4 tư +5 năm +6 sáu +7 bảy +8 tám +9 chín +10 mười +11 mười một +12 mười hai +01 một +02 hai +03 ba +04 tư +05 năm +06 sáu +07 bảy +08 tám +09 chín \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/date/year_suffix.tsv b/nemo_text_processing/text_normalization/vi/data/date/year_suffix.tsv new file mode 100644 index 000000000..31b49f955 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/date/year_suffix.tsv @@ -0,0 +1,4 @@ +tcn trước công nguyên +scn sau công nguyên +TCN trước công nguyên +SCN sau công nguyên \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/fraction/__init__.py b/nemo_text_processing/text_normalization/vi/data/fraction/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/fraction/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/fraction/denominator_exceptions.tsv b/nemo_text_processing/text_normalization/vi/data/fraction/denominator_exceptions.tsv new file mode 100644 index 000000000..7b305e655 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/fraction/denominator_exceptions.tsv @@ -0,0 +1 @@ +4 tư \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/measure/__init__.py b/nemo_text_processing/text_normalization/vi/data/measure/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/measure/base_units.tsv b/nemo_text_processing/text_normalization/vi/data/measure/base_units.tsv new file mode 100644 index 000000000..eb9faf2f5 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/measure/base_units.tsv @@ -0,0 +1,20 @@ +m mét +m2 mét vuông +m3 mét khối +m² mét vuông +m³ mét khối +g gam +l lít +s giây +v vôn +w oát +hz hẹc +A am pe +b bai +B byte +pa pascal +ω ohm +Ω ôm +h giờ +min phút +hr giờ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/measure/measurements_minimal.tsv b/nemo_text_processing/text_normalization/vi/data/measure/measurements_minimal.tsv new file mode 100644 index 000000000..66c0a166b --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/measure/measurements_minimal.tsv @@ -0,0 +1,33 @@ +°f độ f +°F độ F +°c độ c +°C độ C +°k độ k +°K độ K +° độ +°E độ đông +°N độ bắc +°S độ nam +°W độ tây +ha héc ta +mi mile +ft foot +inch inch +yd yard +% phần trăm +hp mã lực +rad radian +kwh ki lô oát giờ +kbps kilobit trên giây +mbps megabit trên giây +ghz gi ga hẹc +mhz mê ga hẹc +tw tê ra oát +kcal ki lô calo +gb gi ga bai +mb mê ga bai +mV mi li vôn +MV mê ga vôn +tb terabyte +pb petabyte +g gam \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/measure/prefixes.tsv b/nemo_text_processing/text_normalization/vi/data/measure/prefixes.tsv new file mode 100644 index 000000000..649ce73a7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/measure/prefixes.tsv @@ -0,0 +1,17 @@ +k ki lô +M mê ga +G gi ga +T tê ra +P pê ta +E ex xa +h hếc tô +da đề ca +d đề xi +c xăng ti +m mi li +µ mi crô +μ mi cờ rô +n na nô +p pi cô +f fem tô +a át tô \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/money/__init__.py b/nemo_text_processing/text_normalization/vi/data/money/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/money/currency.tsv b/nemo_text_processing/text_normalization/vi/data/money/currency.tsv new file mode 100644 index 000000000..95e1eb2ca --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/money/currency.tsv @@ -0,0 +1,51 @@ +$ đô la +đô la đô la +€ ơ rô +¥ yên nhật +₩ won +₫ đồng +đ đồng +đồng đồng +£ bảng anh +₹ rupee +¢ xu +₣ franc pháp +cent xu +cents xu +CHF franc thụy sĩ +FF franc pháp +JPY yên nhật +KRW won hàn quốc +CNY nhân dân tệ +USD đô la +usd đô la +SGD đô la singapore +MYR ringgit malaysia +THB baht thái lan +IDR rupiah indonesia +PHP peso philippines +AUD đô la úc +CAD đô la canada +NZD đô la new zealand +HKD đô la hồng kông +TWD đô la đài loan +ff franc pháp +chf franc thụy sĩ +jpy yên nhật +krw won hàn quốc +cny nhân dân tệ +usd đô la +vnd đồng +vnđ đồng +sgd đô la singapore +myr ringgit malaysia +thb baht thái lan +idr rupiah indonesia +php peso philippines +aud đô la úc +cad đô la canada +nzd đô la new zealand +hkd đô la hồng kông +twd đô la đài loan +VND đồng +VNĐ đồng \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/money/currency_minor.tsv b/nemo_text_processing/text_normalization/vi/data/money/currency_minor.tsv new file mode 100644 index 000000000..d83858714 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/money/currency_minor.tsv @@ -0,0 +1,7 @@ +$ xu +€ xu +£ penny +¢ xu +cent xu +cents xu +pence penny \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/money/per_unit_bases.tsv b/nemo_text_processing/text_normalization/vi/data/money/per_unit_bases.tsv new file mode 100644 index 000000000..feb1808d6 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/money/per_unit_bases.tsv @@ -0,0 +1,8 @@ +g gam +m mét +m² mét vuông +m2 mét vuông +m³ mét khối +m3 mét khối +l lít +B bai \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/money/per_unit_non_metric.tsv b/nemo_text_processing/text_normalization/vi/data/money/per_unit_non_metric.tsv new file mode 100644 index 000000000..c1ccbdf69 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/money/per_unit_non_metric.tsv @@ -0,0 +1,28 @@ +/giờ trên giờ +/h trên giờ +/ngày trên ngày +/d trên ngày +/tuần trên tuần +/tháng trên tháng +/năm trên năm +/phút trên phút +/p trên phút +/giây trên giây +/s trên giây +/lần một lần +/cái một cái +/chiếc một chiếc +/người một người +/chỗ một chỗ +/bài một bài +/trang một trang +/từ một từ +/đồng một đồng +/đêm một đêm +/buổi một buổi +/ca một ca +/dự án một dự án +/lớp một lớp +/khóa một khóa +/suất một suất +/tấn một tấn \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/money/per_unit_prefixes.tsv b/nemo_text_processing/text_normalization/vi/data/money/per_unit_prefixes.tsv new file mode 100644 index 000000000..154aa7306 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/money/per_unit_prefixes.tsv @@ -0,0 +1,6 @@ +k ki lô +M mê ga +G gi ga +c xăng ti +m mi li +T tê ra \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/__init__.py b/nemo_text_processing/text_normalization/vi/data/numbers/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv new file mode 100644 index 000000000..573c20bd4 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +1 một +2 hai +3 ba +4 bốn +5 năm +6 sáu +7 bảy +8 tám +9 chín \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv new file mode 100644 index 000000000..3c3421528 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/digit_special.tsv @@ -0,0 +1,3 @@ +1 một mốt +4 bốn tư +5 năm lăm \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv new file mode 100644 index 000000000..da60cb686 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/magnitudes.tsv @@ -0,0 +1,8 @@ +thousand nghìn +million triệu +billion tỷ +trillion nghìn tỷ +quadrillion triệu tỷ +quintillion tỷ tỷ +hundred trăm +linh linh \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/quantity_abbr.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/quantity_abbr.tsv new file mode 100644 index 000000000..9660bf9bb --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/quantity_abbr.tsv @@ -0,0 +1,7 @@ +k nghìn +K nghìn +tr triệu +TR triệu +Tr triệu +t tỷ +T tỷ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv new file mode 100644 index 000000000..8d99f8a69 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/teen.tsv @@ -0,0 +1,10 @@ +10 mười +11 mười một +12 mười hai +13 mười ba +14 mười bốn +15 mười lăm +16 mười sáu +17 mười bảy +18 mười tám +19 mười chín \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv new file mode 100644 index 000000000..da88b8ab8 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/ties.tsv @@ -0,0 +1,8 @@ +2 hai mươi +3 ba mươi +4 bốn mươi +5 năm mươi +6 sáu mươi +7 bảy mươi +8 tám mươi +9 chín mươi \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/vi/data/numbers/zero.tsv new file mode 100644 index 000000000..df062e38c --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/numbers/zero.tsv @@ -0,0 +1 @@ +0 không \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/vi/data/ordinal/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/ordinal/ordinal_exceptions.tsv b/nemo_text_processing/text_normalization/vi/data/ordinal/ordinal_exceptions.tsv new file mode 100644 index 000000000..5aae90801 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/ordinal/ordinal_exceptions.tsv @@ -0,0 +1,2 @@ +1 nhất +4 tư \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/roman/__init__.py b/nemo_text_processing/text_normalization/vi/data/roman/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/roman/key_word.tsv b/nemo_text_processing/text_normalization/vi/data/roman/key_word.tsv new file mode 100644 index 000000000..73b822342 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/roman/key_word.tsv @@ -0,0 +1,16 @@ +thế kỉ +thế kỷ +thứ +chương +phần +mục +đoạn +năm +khoản +phụ lục +khóa +số +điều +tiểu mục +bài +khối \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/roman/roman_numerals.tsv b/nemo_text_processing/text_normalization/vi/data/roman/roman_numerals.tsv new file mode 100644 index 000000000..d4d8ad20b --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/roman/roman_numerals.tsv @@ -0,0 +1,13 @@ +I 1 +V 5 +X 10 +L 50 +C 100 +D 500 +M 1000 +IV 4 +IX 9 +XL 40 +XC 90 +CD 400 +CM 900 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/time/__init__.py b/nemo_text_processing/text_normalization/vi/data/time/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/time/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/data/time/time_units.tsv b/nemo_text_processing/text_normalization/vi/data/time/time_units.tsv new file mode 100644 index 000000000..66e912f4e --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/time/time_units.tsv @@ -0,0 +1,4 @@ +h giờ +g giờ +p phút +s giây \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/time/time_zones.tsv b/nemo_text_processing/text_normalization/vi/data/time/time_zones.tsv new file mode 100644 index 000000000..74c39849a --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/data/time/time_zones.tsv @@ -0,0 +1,18 @@ +GMT GMT +UTC UTC +CST CST +PST PST +EST EST +JST JST +PT PT +ET ET +CET CET +gmt GMT +utc UTC +cst CST +pst PST +est EST +jst JST +pt PT +et ET +cet CET \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/vi/data/whitelist.tsv b/nemo_text_processing/text_normalization/vi/data/whitelist.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/nemo_text_processing/text_normalization/vi/graph_utils.py b/nemo_text_processing/text_normalization/vi/graph_utils.py new file mode 100644 index 000000000..1c0c1a0ab --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/graph_utils.py @@ -0,0 +1,165 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_COMMA = "," +NEMO_COMMA_VI = "phẩy" + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + +quoted_text = pynini.closure(NEMO_NOT_QUOTE) + + +def extract_field(field_name): + return pynutil.delete(f"{field_name}:") + delete_space + pynutil.delete("\"") + quoted_text + pynutil.delete("\"") + + +def extract_wrapper_content(wrapper_type: str, content_graph): + """Helper to extract content from wrapper like 'decimal { ... }'""" + return pynutil.delete(f"{wrapper_type} {{") + delete_space + content_graph + delete_space + pynutil.delete("}") + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/vi/taggers/__init__.py b/nemo_text_processing/text_normalization/vi/taggers/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/taggers/cardinal.py b/nemo_text_processing/text_normalization/vi/taggers/cardinal.py new file mode 100644 index 000000000..0e0fd17ec --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/cardinal.py @@ -0,0 +1,244 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_DIGIT, GraphFst, insert_space +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class CardinalFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + resources = { + 'zero': pynini.string_file(get_abs_path("data/numbers/zero.tsv")), + 'digit': pynini.string_file(get_abs_path("data/numbers/digit.tsv")), + 'teen': pynini.string_file(get_abs_path("data/numbers/teen.tsv")), + 'ties': pynini.string_file(get_abs_path("data/numbers/ties.tsv")), + } + self.zero, self.digit, self.teen, self.ties = resources.values() + + magnitudes_labels = load_labels(get_abs_path("data/numbers/magnitudes.tsv")) + self.magnitudes = {parts[0]: parts[1] for parts in magnitudes_labels if len(parts) == 2} + + digit_special_labels = load_labels(get_abs_path("data/numbers/digit_special.tsv")) + special = {parts[0]: {'std': parts[1], 'alt': parts[2]} for parts in digit_special_labels if len(parts) >= 3} + + self.special_digits = pynini.union( + *[pynini.cross(k, v["alt"]) for k, v in special.items() if k in ["1", "4", "5"]] + ) + self.linh_digits = pynini.union(*[pynini.cross(k, special[k]["std"]) for k in ["1", "4", "5"]], self.digit) + + self.two_digit = pynini.union( + self.teen, + self.ties + pynutil.delete("0"), + self.ties + + insert_space + + pynini.union(self.special_digits, pynini.union("2", "3", "6", "7", "8", "9") @ self.digit), + ) + + hundred_word = self.magnitudes["hundred"] + linh_word = self.magnitudes["linh"] + + # X00: một trăm, hai trăm, etc. + hundreds_exact = self.digit + insert_space + pynutil.insert(hundred_word) + pynutil.delete("00") + + # X0Y: một trăm linh một, hai trăm linh năm, etc. + hundreds_with_linh = ( + self.digit + + insert_space + + pynutil.insert(hundred_word) + + pynutil.delete("0") + + insert_space + + pynutil.insert(linh_word) + + insert_space + + self.linh_digits + ) + + # XYZ: một trăm hai mười ba, etc. + hundreds_with_tens = self.digit + insert_space + pynutil.insert(hundred_word) + insert_space + self.two_digit + + # 0YZ: Handle numbers starting with 0 (e.g., 087 -> tám mươi bảy) + leading_zero_tens = pynutil.delete("0") + self.two_digit + + # 00Z: Handle numbers starting with 00 (e.g., 008 -> tám) + leading_double_zero = pynutil.delete("00") + self.digit + + self.hundreds_pattern = pynini.union( + hundreds_exact, + hundreds_with_linh, + hundreds_with_tens, + leading_zero_tens, + leading_double_zero, + ) + + self.hundreds = pynini.closure(NEMO_DIGIT, 3, 3) @ self.hundreds_pattern + + self.magnitude_patterns = self._build_all_magnitude_patterns() + custom_patterns = self._build_all_patterns() + + all_patterns = [ + *custom_patterns, + *self.magnitude_patterns.values(), + self.hundreds, + self.two_digit, + self.digit, + self.zero, + ] + self.graph = pynini.union(*all_patterns).optimize() + + self.single_digits_graph = self.digit | self.zero + + negative = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + final_graph = negative + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + self.fst = self.add_tokens(final_graph).optimize() + + def _build_magnitude_pattern(self, name, min_digits, max_digits, zero_count, prev_pattern=None): + magnitude_word = self.magnitudes[name] + linh_word = self.magnitudes["linh"] + patterns = [] + + for digits in range(min_digits, max_digits + 1): + leading_digits = digits - zero_count + if leading_digits == 1: + leading_fst = self.digit + elif leading_digits == 2: + leading_fst = self.two_digit + else: + leading_fst = self.hundreds_pattern + + prefix = leading_fst + insert_space + pynutil.insert(magnitude_word) + digit_patterns = [prefix + pynutil.delete("0" * zero_count)] + + if prev_pattern and name not in ["quadrillion", "quintillion"]: + digit_patterns.append(prefix + insert_space + prev_pattern) + + for trailing_zeros in range(zero_count): + remaining_digits = zero_count - trailing_zeros + trailing_prefix = prefix + pynutil.delete("0" * trailing_zeros) + + if remaining_digits == 1: + linh_pattern = ( + trailing_prefix + insert_space + pynutil.insert(linh_word) + insert_space + self.linh_digits + ) + digit_patterns.append(pynutil.add_weight(linh_pattern, -0.1)) + elif remaining_digits == 2: + digit_patterns.append(trailing_prefix + insert_space + self.two_digit) + elif remaining_digits == 3: + digit_patterns.append(trailing_prefix + insert_space + self.hundreds_pattern) + + patterns.append(pynini.closure(NEMO_DIGIT, digits, digits) @ pynini.union(*digit_patterns)) + + return pynini.union(*patterns) + + def _build_all_magnitude_patterns(self): + magnitude_config = [ + ("thousand", 4, 6, 3), + ("million", 7, 9, 6), + ("billion", 10, 12, 9), + ("trillion", 13, 15, 12), + ("quadrillion", 16, 18, 15), + ("quintillion", 19, 21, 18), + ] + patterns = {} + prev_pattern = None + for name, min_digits, max_digits, zero_count in magnitude_config: + if name in self.magnitudes: + patterns[name] = self._build_magnitude_pattern(name, min_digits, max_digits, zero_count, prev_pattern) + prev_pattern = patterns[name] + else: + break + return patterns + + def _get_zero_or_magnitude_pattern(self, digits, magnitude_key): + """Create pattern that handles all-zeros or normal magnitude processing""" + all_zeros = "0" * digits + return pynini.union(pynini.cross(all_zeros, ""), NEMO_DIGIT**digits @ self.magnitude_patterns[magnitude_key]) + + def _build_all_patterns(self): + patterns = [] + delete_dot = pynutil.delete(".") + + # Large number split patterns (>12 digits): front + "tỷ" + back(9 digits) + if "billion" in self.magnitudes: + billion_word = self.magnitudes["billion"] + back_digits = 9 + + for total_digits in range(13, 22): + front_digits = total_digits - back_digits + front_pattern = self._get_pattern_for_digits(front_digits) + if front_pattern: + back_pattern = self._get_zero_or_magnitude_pattern(back_digits, "million") + split_pattern = ( + front_pattern + insert_space + pynutil.insert(billion_word) + insert_space + back_pattern + ) + patterns.append(NEMO_DIGIT**total_digits @ pynutil.add_weight(split_pattern, -0.5)) + + # Dot patterns + dot_configs = [(6, None), (5, None), (4, None), (3, "billion"), (2, "million"), (1, "thousand")] + for dots, magnitude in dot_configs: + pattern = (NEMO_DIGIT - "0") + pynini.closure(NEMO_DIGIT, 0, 2) + for _ in range(dots): + pattern += delete_dot + NEMO_DIGIT**3 + + if magnitude and magnitude in self.magnitude_patterns: + patterns.append(pynini.compose(pynutil.add_weight(pattern, -0.3), self.magnitude_patterns[magnitude])) + elif not magnitude: + if dots == 4: + digit_range = [13, 14, 15] + elif dots == 5: + digit_range = [16, 17, 18] + elif dots == 6: + digit_range = [19, 20, 21] + else: + digit_range = [] + + for digit_count in digit_range: + if 13 <= digit_count <= 21: + front_digits = digit_count - back_digits + front_pattern = self._get_pattern_for_digits(front_digits) + if front_pattern: + back_pattern = self._get_zero_or_magnitude_pattern(back_digits, "million") + split = ( + (NEMO_DIGIT**front_digits @ front_pattern) + + insert_space + + pynutil.insert(self.magnitudes["billion"]) + + insert_space + + back_pattern + ) + patterns.append( + pynini.compose(pattern, NEMO_DIGIT**digit_count @ pynutil.add_weight(split, -1.0)) + ) + + return patterns + + def _get_pattern_for_digits(self, digit_count): + if digit_count <= 0: + return None + elif digit_count == 1: + return self.digit + elif digit_count == 2: + return self.two_digit + elif digit_count == 3: + return self.hundreds_pattern + elif digit_count <= 6: + return self.magnitude_patterns.get("thousand") + elif digit_count <= 9: + return self.magnitude_patterns.get("million") + elif digit_count <= 12: + return self.magnitude_patterns.get("billion") + else: + return None diff --git a/nemo_text_processing/text_normalization/vi/taggers/date.py b/nemo_text_processing/text_normalization/vi/taggers/date.py new file mode 100644 index 000000000..c8b18bf7b --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/date.py @@ -0,0 +1,168 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class DateFst(GraphFst): + """ + Finite state transducer for classifying Vietnamese dates, e.g. + 15/01/2024 -> date { day: "mười lăm" month: "một" year: "hai nghìn hai mươi tư" } + tháng 4 2024 -> date { month: "tư" year: "hai nghìn hai mươi tư" } + ngày 15/01/2024 -> date { day: "mười lăm" month: "một" year: "hai nghìn hai mươi tư" } + ngày 12 tháng 5 năm 2025 -> date { day: "mười hai" month: "năm" year: "hai nghìn hai mươi lăm" } + năm 20 SCN -> date { year: "hai mươi" era: "sau công nguyên" } + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="date", kind="classify", deterministic=deterministic) + + # Vietnamese date keywords + DAY_WORD = "ngày" + MONTH_WORD = "tháng" + YEAR_WORD = "năm" + ORDINAL_YEAR_WORD = "năm thứ" + + # Prebuilt patterns for common usage + day_prefix = pynini.accep(DAY_WORD + NEMO_SPACE) + month_prefix = pynini.accep(MONTH_WORD + NEMO_SPACE) + year_prefix = pynini.accep(YEAR_WORD + NEMO_SPACE) + ordinal_year_prefix = pynini.accep(ORDINAL_YEAR_WORD + NEMO_SPACE) + + delete_day_prefix = pynutil.delete(DAY_WORD + NEMO_SPACE) + delete_month_prefix = pynutil.delete(MONTH_WORD + NEMO_SPACE) + delete_year_prefix = pynutil.delete(YEAR_WORD + NEMO_SPACE) + delete_ordinal_year_prefix = pynutil.delete(ORDINAL_YEAR_WORD + NEMO_SPACE) + + day_mappings = load_labels(get_abs_path("data/date/days.tsv")) + month_mappings = load_labels(get_abs_path("data/date/months.tsv")) + era_mappings = load_labels(get_abs_path("data/date/year_suffix.tsv")) + + one_or_two_digits = pynini.closure(NEMO_DIGIT, 1, 2) + year_digit = pynini.closure(NEMO_DIGIT, 1, 4) + separator = pynini.union("/", "-", ".") + + day_convert = pynini.string_map([(k, v) for k, v in day_mappings]) + month_convert = pynini.string_map([(k, v) for k, v in month_mappings]) + year_convert = pynini.compose(year_digit, cardinal.graph) + + era_to_full = {} + for abbr, full_form in era_mappings: + era_to_full[abbr.lower()] = full_form + era_to_full[abbr.upper()] = full_form + + era_convert = pynini.string_map([(k, v) for k, v in era_to_full.items()]) + + day_part = pynutil.insert("day: \"") + day_convert + pynutil.insert("\" ") + month_part = pynutil.insert("month: \"") + month_convert + pynutil.insert("\" ") + year_part = pynutil.insert("year: \"") + year_convert + pynutil.insert("\"") + month_final = pynutil.insert("month: \"") + month_convert + pynutil.insert("\"") + era_part = pynutil.insert("era: \"") + era_convert + pynutil.insert("\"") + + patterns = [] + + # DD/MM/YYYY format (Vietnamese standard) + date_sep = day_part + pynutil.delete(separator) + month_part + pynutil.delete(separator) + year_part + patterns.append( + pynini.compose(one_or_two_digits + separator + one_or_two_digits + separator + year_digit, date_sep) + ) + patterns.append( + pynini.compose( + day_prefix + one_or_two_digits + separator + one_or_two_digits + separator + year_digit, + delete_day_prefix + date_sep, + ) + ) + + # YYYY/MM/DD format (ISO standard) - output in Vietnamese order + iso_year_part = pynutil.insert("year: \"") + year_convert + pynutil.insert("\" ") + iso_month_part = pynutil.insert("month: \"") + month_convert + pynutil.insert("\" ") + iso_day_part = pynutil.insert("day: \"") + day_convert + pynutil.insert("\"") + + iso_date_sep = ( + iso_year_part + pynutil.delete(separator) + iso_month_part + pynutil.delete(separator) + iso_day_part + ) + patterns.append( + pynini.compose(year_digit + separator + one_or_two_digits + separator + one_or_two_digits, iso_date_sep) + ) + + for sep in [separator, pynini.accep(NEMO_SPACE)]: + patterns.append( + pynini.compose( + month_prefix + one_or_two_digits + sep + year_digit, + delete_month_prefix + month_part + pynutil.delete(sep) + year_part, + ) + ) + + day_month_sep = day_part + pynutil.delete(separator) + month_final + patterns.append( + pynini.compose( + day_prefix + one_or_two_digits + separator + one_or_two_digits, delete_day_prefix + day_month_sep + ) + ) + + patterns.append( + pynini.compose( + day_prefix + + one_or_two_digits + + pynini.accep(NEMO_SPACE + MONTH_WORD + NEMO_SPACE) + + one_or_two_digits, + delete_day_prefix + day_part + pynutil.delete(NEMO_SPACE + MONTH_WORD + NEMO_SPACE) + month_final, + ) + ) + + patterns.append( + pynini.compose( + day_prefix + + one_or_two_digits + + pynini.accep(NEMO_SPACE + MONTH_WORD + NEMO_SPACE) + + one_or_two_digits + + pynini.accep(NEMO_SPACE + YEAR_WORD + NEMO_SPACE) + + year_digit, + delete_day_prefix + + day_part + + pynutil.delete(NEMO_SPACE + MONTH_WORD + NEMO_SPACE) + + month_part + + pynutil.delete(NEMO_SPACE + YEAR_WORD + NEMO_SPACE) + + year_part, + ) + ) + + patterns.append(pynini.compose(year_prefix + year_digit, delete_year_prefix + year_part)) + + era_abbrs = list(era_to_full.keys()) + for era_abbr in era_abbrs: + patterns.append( + pynini.compose( + year_prefix + year_digit + pynini.accep(NEMO_SPACE) + pynini.accep(era_abbr), + delete_year_prefix + year_part + pynutil.delete(NEMO_SPACE) + era_part, + ) + ) + + patterns.append( + pynini.compose( + ordinal_year_prefix + year_digit + pynini.accep(NEMO_SPACE) + pynini.accep(era_abbr), + delete_ordinal_year_prefix + + pynutil.insert("ordinal: \"") + + year_convert + + pynutil.insert("\" ") + + pynutil.delete(NEMO_SPACE) + + era_part, + ) + ) + + self.fst = self.add_tokens(pynini.union(*patterns)) diff --git a/nemo_text_processing/text_normalization/vi/taggers/decimal.py b/nemo_text_processing/text_normalization/vi/taggers/decimal.py new file mode 100644 index 000000000..d5591a073 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/decimal.py @@ -0,0 +1,165 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_COMMA, NEMO_DIGIT, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class DecimalFst(GraphFst): + """ + Finite state transducer for classifying Vietnamese decimal numbers, e.g. + -12,5 tỷ -> decimal { negative: "true" integer_part: "mười hai" fractional_part: "năm" quantity: "tỷ" } + 12.345,67 -> decimal { integer_part: "mười hai nghìn ba trăm bốn mươi lăm" fractional_part: "sáu bảy" } + 1tr2 -> decimal { integer_part: "một triệu hai trăm nghìn" } + 818,303 -> decimal { integer_part: "tám trăm mười tám" fractional_part: "ba không ba" } + 0,2 triệu -> decimal { integer_part: "không" fractional_part: "hai" quantity: "triệu" } + Args: + cardinal: CardinalFst instance for processing integer parts + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="decimal", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + self.graph = cardinal.single_digits_graph.optimize() + if not deterministic: + self.graph = self.graph | cardinal_graph + + # Load data + digit_labels = load_labels(get_abs_path("data/numbers/digit.tsv")) + zero_labels = load_labels(get_abs_path("data/numbers/zero.tsv")) + magnitude_labels = load_labels(get_abs_path("data/numbers/magnitudes.tsv")) + quantity_abbr_labels = load_labels(get_abs_path("data/numbers/quantity_abbr.tsv")) + + # Common components + single_digit_map = pynini.union(*[pynini.cross(k, v) for k, v in digit_labels + zero_labels]) + quantity_units = pynini.union(*[v for _, v in magnitude_labels]) + one_to_three_digits = pynini.closure(NEMO_DIGIT, 1, 3) + + # Building blocks + integer_part = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"") + fractional_part = ( + pynutil.insert("fractional_part: \"") + + single_digit_map + + pynini.closure(pynutil.insert(NEMO_SPACE) + single_digit_map) + + pynutil.insert("\"") + ) + optional_quantity = ( + pynutil.delete(NEMO_SPACE).ques + pynutil.insert(" quantity: \"") + quantity_units + pynutil.insert("\"") + ).ques + + patterns = [] + + # 1. Basic decimal patterns: 12,5 and 12,5 tỷ + basic_decimal = ( + (integer_part + pynutil.insert(NEMO_SPACE)).ques + + pynutil.delete(NEMO_COMMA) + + pynutil.insert(NEMO_SPACE) + + fractional_part + ) + patterns.append(basic_decimal) + patterns.append(basic_decimal + optional_quantity) + + # 2. Thousand-separated decimals: 12.345,67 and 12.345,67 tỷ + integer_with_dots = ( + NEMO_DIGIT + pynini.closure(NEMO_DIGIT, 0, 2) + pynini.closure(pynutil.delete(".") + NEMO_DIGIT**3, 1) + ) + separated_integer_part = ( + pynutil.insert("integer_part: \"") + + pynini.compose(integer_with_dots, cardinal_graph) + + pynutil.insert("\"") + ) + separated_decimal = ( + separated_integer_part + + pynutil.insert(NEMO_SPACE) + + pynutil.delete(NEMO_COMMA) + + pynutil.insert(NEMO_SPACE) + + fractional_part + ) + patterns.append(separated_decimal) + patterns.append(separated_decimal + optional_quantity) + + # 3. Integer with quantity: 100 triệu + integer_with_quantity = ( + integer_part + + pynutil.delete(NEMO_SPACE).ques + + pynutil.insert(" quantity: \"") + + quantity_units + + pynutil.insert("\"") + ) + patterns.append(integer_with_quantity) + + # 4. Standard abbreviations: 1k, 100tr, etc. + for abbr, full_name in quantity_abbr_labels: + abbr_pattern = pynini.compose( + one_to_three_digits + pynutil.delete(abbr), + pynutil.insert("integer_part: \"") + + pynini.compose(one_to_three_digits, cardinal_graph) + + pynutil.insert(f"\" quantity: \"{full_name}\""), + ) + patterns.append(abbr_pattern) + + # 5. Decimal with abbreviations: 2,5tr, but avoid measure conflicts + measure_prefix_labels = load_labels(get_abs_path("data/measure/prefixes.tsv")) + measure_prefixes = {prefix.lower() for prefix, _ in measure_prefix_labels} + + # Filter quantity abbreviations to avoid measure conflicts + safe_quantity_abbrs = [ + (abbr, full) for abbr, full in quantity_abbr_labels if abbr.lower() not in measure_prefixes + ] + + for abbr, full_name in safe_quantity_abbrs: + decimal_abbr_pattern = ( + (integer_part + pynutil.insert(NEMO_SPACE)).ques + + pynutil.delete(NEMO_COMMA) + + pynutil.insert(NEMO_SPACE) + + fractional_part + + pynutil.insert(f" quantity: \"{full_name}\"") + + pynutil.delete(abbr) + ) + patterns.append(decimal_abbr_pattern) + + # 6. Compound abbreviations: 1tr2 -> một triệu hai trăm nghìn, 2t3 -> hai tỷ ba trăm triệu + compound_expansions = { + "tr": ("triệu", "trăm nghìn"), # 1tr2 -> một triệu hai trăm nghìn + "t": ("tỷ", "trăm triệu"), # 2t3 -> hai tỷ ba trăm triệu + } + + for abbr, (major_unit, minor_suffix) in compound_expansions.items(): + pattern = one_to_three_digits + pynini.cross(abbr, "") + NEMO_DIGIT + expansion = ( + pynutil.insert("integer_part: \"") + + pynini.compose(one_to_three_digits, cardinal_graph) + + pynutil.insert(f" {major_unit} ") + + pynini.compose(NEMO_DIGIT, cardinal_graph) + + pynutil.insert(f" {minor_suffix}\"") + ) + patterns.append(pynini.compose(pattern, expansion)) + + # Combine all patterns + self._final_graph_wo_negative = pynini.union(*patterns).optimize() + + # Add optional negative prefix + negative = (pynutil.insert("negative: ") + pynini.cross("-", "\"true\" ")).ques + final_graph = negative + self._final_graph_wo_negative + + self.fst = self.add_tokens(final_graph).optimize() + + @property + def final_graph_wo_negative(self): + """Graph without negative prefix, used by MoneyFst""" + return self._final_graph_wo_negative diff --git a/nemo_text_processing/text_normalization/vi/taggers/fraction.py b/nemo_text_processing/text_normalization/vi/taggers/fraction.py new file mode 100644 index 000000000..c54f5be5f --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/fraction.py @@ -0,0 +1,72 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.vi.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class FractionFst(GraphFst): + """ + Finite state transducer for classifying Vietnamese fraction numbers, e.g. + 23 1/5 -> fraction { integer_part: "hai mươi ba" numerator: "một" denominator: "năm" } + 3/9 -> fraction { numerator: "ba" denominator: "chín" } + 1/4 -> fraction { numerator: "một" denominator: "tư" } + + Args: + cardinal: CardinalFst for converting numbers to Vietnamese words + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal: CardinalFst, deterministic: bool = True): + super().__init__(name="fraction", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + digit = pynini.union(*[str(i) for i in range(10)]) + number = pynini.closure(digit, 1) + + denominator_exceptions = { + row[0]: row[1] for row in load_labels(get_abs_path("data/fraction/denominator_exceptions.tsv")) + } + + denominator_exception_patterns = [pynini.cross(k, v) for k, v in denominator_exceptions.items()] + denominator_exception_graph = ( + pynini.union(*denominator_exception_patterns) if denominator_exception_patterns else None + ) + denominator_graph = ( + pynini.union(denominator_exception_graph, cardinal_graph) + if denominator_exception_graph + else cardinal_graph + ) + + numerator = ( + pynutil.insert("numerator: \"") + (number @ cardinal_graph) + pynutil.insert("\" ") + pynutil.delete("/") + ) + denominator = pynutil.insert("denominator: \"") + (number @ denominator_graph) + pynutil.insert("\"") + integer_part = pynutil.insert("integer_part: \"") + (number @ cardinal_graph) + pynutil.insert("\" ") + + simple_fraction = numerator + denominator + mixed_fraction = integer_part + pynutil.delete(NEMO_SPACE) + numerator + denominator + + # Create graph without negative for reuse in other FSTs (like measure) + fraction_wo_negative = simple_fraction | mixed_fraction + self.final_graph_wo_negative = fraction_wo_negative.optimize() + + optional_graph_negative = (pynutil.insert("negative: ") + pynini.cross("-", "\"true\" ")).ques + + self.fst = self.add_tokens(optional_graph_negative + (simple_fraction | mixed_fraction)).optimize() diff --git a/nemo_text_processing/text_normalization/vi/taggers/measure.py b/nemo_text_processing/text_normalization/vi/taggers/measure.py new file mode 100644 index 000000000..8de1d3632 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/measure.py @@ -0,0 +1,153 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_COMMA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + delete_space, +) +from nemo_text_processing.text_normalization.vi.utils import get_abs_path + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying measure for Vietnamese, e.g. + 12kg -> measure { cardinal { integer: "mười hai" } units: "ki lô gam" } + 1kg -> measure { cardinal { integer: "một" } units: "ki lô gam" } + 0.5kg -> measure { decimal { fractional_part: "năm" } units: "ki lô gam" } + -12kg -> measure { negative: "true" cardinal { integer: "mười hai" } units: "ki lô gam" } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def _create_measure_subgraph(self, measure_type: str, number_graph, optional_negative, graph_unit): + """Helper to create measure subgraph pattern - reduces duplication""" + return ( + optional_negative + + pynutil.insert(f"{measure_type} {{ ") + + number_graph + + pynutil.insert(" }") + + delete_space + + pynutil.insert(" units: \"") + + graph_unit + + pynutil.insert('"') + ) + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + cardinal_graph = cardinal.graph + + # Load minimal measurement files (massive redundancy removed via subfst) + measurements_path = get_abs_path("data/measure/measurements_minimal.tsv") + prefixes_path = get_abs_path("data/measure/prefixes.tsv") + base_units_path = get_abs_path("data/measure/base_units.tsv") + + # Create subfst for metric units: prefix + space + base_unit + graph_prefixes = pynini.string_file(prefixes_path) + graph_base_units = pynini.string_file(base_units_path) + space = pynutil.insert(NEMO_SPACE) + graph_metric_units = graph_prefixes + space + graph_base_units + + # Load non-metric and special units + graph_special_units = pynini.string_file(measurements_path) + + # Also allow base units without prefixes (e.g., 'g' not just 'kg') + graph_standalone_units = graph_base_units + + # Combine all unit mappings + graph_unit = graph_metric_units | graph_special_units | graph_standalone_units + + # Add compound unit support (unit/unit patterns like km/h) + graph_unit_compound = pynini.cross("/", " trên ") + pynutil.insert(NEMO_SPACE) + graph_unit + + optional_graph_unit_compound = pynini.closure( + pynutil.insert(NEMO_SPACE) + graph_unit_compound, + 0, + 1, + ) + + # Update unit graph to include compound units + graph_unit = graph_unit + optional_graph_unit_compound | graph_unit_compound + + # Create unit symbol pattern using FST operations (no loops needed) + prefix_symbols = pynini.project(graph_prefixes, "input") # Extract prefix symbols + base_symbols = pynini.project(graph_base_units, "input") # Extract base symbols + special_symbols = pynini.project(graph_special_units, "input") # Extract special symbols + + # Build unit pattern: metric combinations | standalone bases | special units + metric_pattern = prefix_symbols + base_symbols # All prefix+base combinations + simple_unit_pattern = metric_pattern | base_symbols | special_symbols + + # Add compound unit patterns to recognition + compound_pattern = simple_unit_pattern + "/" + simple_unit_pattern + unit_pattern = simple_unit_pattern | compound_pattern + + number = pynini.closure(NEMO_DIGIT, 1) + decimal_number = number + NEMO_COMMA + number + + # Optional negative sign handling for Vietnamese + optional_graph_negative = pynini.closure( + pynini.cross("-", "negative: \"true\" "), + 0, + 1, + ) + + # Domain restriction patterns - only match core number+unit patterns + # Remove punctuation handling to let punctuation tagger handle it separately + optional_space = pynini.closure(NEMO_SPACE, 0, 1) + optional_negative_sign = pynini.closure("-" + optional_space, 0, 1) + + integer_measure_domain = optional_negative_sign + number + optional_space + unit_pattern + decimal_measure_domain = optional_negative_sign + decimal_number + optional_space + unit_pattern + fraction_measure_domain = optional_negative_sign + number + "/" + number + optional_space + unit_pattern + + cardinal_number_graph = pynutil.insert('integer: "') + (number @ cardinal_graph) + pynutil.insert('"') + + subgraph_cardinal = self._create_measure_subgraph( + "cardinal", cardinal_number_graph, optional_graph_negative, graph_unit + ) + subgraph_decimal = self._create_measure_subgraph( + "decimal", decimal.final_graph_wo_negative, optional_graph_negative, graph_unit + ) + subgraph_fraction = self._create_measure_subgraph( + "fraction", fraction.final_graph_wo_negative, optional_graph_negative, graph_unit + ) + + # Apply domain restrictions to ensure we only match complete number+unit patterns + subgraph_cardinal = pynini.compose(integer_measure_domain, subgraph_cardinal) + subgraph_decimal = pynini.compose(decimal_measure_domain, subgraph_decimal) + subgraph_fraction = pynini.compose(fraction_measure_domain, subgraph_fraction) + + # Final graph combining main patterns + final_graph = subgraph_cardinal | subgraph_decimal | subgraph_fraction + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/vi/taggers/money.py b/nemo_text_processing/text_normalization/vi/taggers/money.py new file mode 100644 index 000000000..540094591 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/money.py @@ -0,0 +1,198 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_COMMA, + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + convert_space, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying money, e.g. + "10,5$" -> money { integer_part: "mười" currency_maj: "đô la" fractional_part: "năm mươi" currency_min: "xu" preserve_order: true } + "10đ" -> money { integer_part: "mười" currency_maj: "đồng" } + "10 triệu đồng" -> money { integer_part: "mười" quantity: "triệu" currency_maj: "đồng" } + + Args: + cardinal: CardinalFst instance for processing integer parts + decimal: DecimalFst instance for processing fractional parts + deterministic: if True will provide a single transduction option, for False multiple transduction are generated. + """ + + def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + # Load data + currency_major_labels = load_labels(get_abs_path("data/money/currency.tsv")) + currency_minor_labels = load_labels(get_abs_path("data/money/currency_minor.tsv")) + quantity_graph = pynini.string_file(get_abs_path("data/numbers/quantity_abbr.tsv")) + + # Load optimized per_unit files using subfst approach + per_unit_non_metric_path = get_abs_path("data/money/per_unit_non_metric.tsv") + per_unit_prefixes_path = get_abs_path("data/money/per_unit_prefixes.tsv") + per_unit_bases_path = get_abs_path("data/money/per_unit_bases.tsv") + + # Create subfst for metric per_unit patterns + graph_prefixes = pynini.string_file(per_unit_prefixes_path) + graph_bases = pynini.string_file(per_unit_bases_path) + + # Build metric combinations: "/kg" -> "một ki lô gam" + slash = pynutil.delete("/") + one_space = pynutil.insert("một ") + space = pynutil.insert(NEMO_SPACE) + + graph_metric_per_units = slash + one_space + graph_prefixes + space + graph_bases + graph_standalone_per_units = slash + one_space + graph_bases + + # Load non-metric per_unit entries + graph_non_metric_per_units = pynini.string_file(per_unit_non_metric_path) + + # Combine all per_unit mappings + per_unit_graph = graph_metric_per_units | graph_standalone_per_units | graph_non_metric_per_units + + # Basic components + cardinal_graph = cardinal.graph + currency_major_graph = pynini.string_map(currency_major_labels) + currency_minor_map = dict(currency_minor_labels) + decimal_graph = decimal.final_graph_wo_negative + + # Common patterns + integer_part = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + preserve_order = pynutil.insert(" preserve_order: true") + optional_space = pynini.closure(delete_space, 0, 1) + + # Fractional part conversion for cents + two_digits_fractional_part = ( + pynini.closure(NEMO_DIGIT) + (NEMO_DIGIT - "0") + pynini.closure(pynutil.delete("0")) + ) @ ( + (pynutil.delete("0") + (NEMO_DIGIT - "0")) + | ((NEMO_DIGIT - "0") + pynutil.insert("0")) + | ((NEMO_DIGIT - "0") + NEMO_DIGIT) + ) + fractional_conversion = two_digits_fractional_part @ cardinal_graph + fractional_part = pynutil.insert('fractional_part: "') + fractional_conversion + pynutil.insert('"') + + all_patterns = [] + + # 1. Symbol-based patterns + symbol_patterns = [] + minor_only_patterns = [] + + for symbol, major_name in currency_major_labels: + maj_tag = pynutil.insert(f' currency_maj: "{major_name}"') + + # Simple integer pattern: 10$ -> mười đô la + simple_pattern = integer_part + pynutil.delete(symbol) + insert_space + maj_tag + symbol_patterns.append(simple_pattern) + + # Patterns with minor currency (cents/xu) + if symbol in currency_minor_map: + minor_name = currency_minor_map[symbol] + min_tag = pynutil.insert(f' currency_min: "{minor_name}"') + + # Minor-only pattern: 0,5$ -> năm mươi xu (highest priority) + minor_only = ( + pynutil.delete("0") + + pynutil.delete(NEMO_COMMA) + + fractional_part + + insert_space + + min_tag + + pynutil.delete(symbol) + + preserve_order + ) + minor_only_patterns.append(minor_only) + + # Major + minor pattern: 10,5$ -> mười đô la năm mươi xu + major_minor = ( + integer_part + + insert_space + + maj_tag + + pynini.cross(NEMO_COMMA, NEMO_SPACE) + + fractional_part + + insert_space + + min_tag + + pynutil.delete(symbol) + + preserve_order + ) + symbol_patterns.append(major_minor) + + # 2. Word-based patterns + word_patterns = [] + + # Complex decimal + currency: 1tr5 vnd -> một triệu năm trăm nghìn đồng + decimal_with_currency = ( + decimal_graph + + optional_space + + insert_space + + pynutil.insert(' currency_maj: "') + + convert_space(currency_major_graph) + + pynutil.insert('"') + ) + word_patterns.append(decimal_with_currency) + + # Quantity + currency: 10tr đồng -> mười triệu đồng + quantity_tag = pynutil.insert(' quantity: "') + convert_space(quantity_graph) + pynutil.insert('"') + quantity_pattern = ( + integer_part + + optional_space + + insert_space + + quantity_tag + + optional_space + + insert_space + + pynutil.insert(' currency_maj: "') + + convert_space(currency_major_graph) + + pynutil.insert('"') + ) + word_patterns.append(quantity_pattern) + + # Simple word pattern: 10 đồng -> mười đồng + simple_word_pattern = ( + integer_part + + optional_space + + insert_space + + pynutil.insert(' currency_maj: "') + + convert_space(currency_major_graph) + + pynutil.insert('"') + ) + word_patterns.append(simple_word_pattern) + + # Combine patterns with priorities + # Minor-only patterns get highest priority (negative weight) + if minor_only_patterns: + all_patterns.append(pynutil.add_weight(pynini.union(*minor_only_patterns), -0.0001)) + + # Symbol patterns get normal priority + if symbol_patterns: + all_patterns.append(pynini.union(*symbol_patterns)) + + # Word patterns get lowest priority + if word_patterns: + all_patterns.append(pynutil.add_weight(pynini.union(*word_patterns), 0.1)) + + # Final graph with optional per-unit support + final_graph = pynini.union(*all_patterns) + per_unit_tag = pynutil.insert(' morphosyntactic_features: "') + per_unit_graph + pynutil.insert('"') + final_graph += per_unit_tag.ques + + self.fst = self.add_tokens(final_graph.optimize()) diff --git a/nemo_text_processing/text_normalization/vi/taggers/ordinal.py b/nemo_text_processing/text_normalization/vi/taggers/ordinal.py new file mode 100644 index 000000000..af8ca3053 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/ordinal.py @@ -0,0 +1,61 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_DIGIT, GraphFst +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for classifying Vietnamese ordinals, e.g. + thứ 1 -> ordinal { integer: "nhất" } + thứ 4 -> ordinal { integer: "tư" } + thứ 15 -> ordinal { integer: "mười lăm" } + Args: + cardinal: CardinalFst for number conversion + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="ordinal", kind="classify", deterministic=deterministic) + + prefix = "thứ " + number_pattern = pynini.closure(NEMO_DIGIT, 1) + + ordinal_exceptions = { + row[0]: row[1] for row in load_labels(get_abs_path("data/ordinal/ordinal_exceptions.tsv")) + } + + exception_patterns = [] + for digit, word in ordinal_exceptions.items(): + exception_patterns.append(pynini.cross(digit, word)) + + exception_graph = pynini.union(*exception_patterns) if exception_patterns else None + + combined_graph = cardinal.graph + if exception_graph: + combined_graph = pynini.union(exception_graph, cardinal.graph) + + self.graph = ( + pynutil.delete(prefix) + + pynutil.insert("integer: \"") + + pynini.compose(number_pattern, combined_graph) + + pynutil.insert("\"") + ) + + self.fst = self.add_tokens(self.graph).optimize() diff --git a/nemo_text_processing/text_normalization/vi/taggers/punctuation.py b/nemo_text_processing/text_normalization/vi/taggers/punctuation.py new file mode 100644 index 000000000..1244f236a --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/punctuation.py @@ -0,0 +1,35 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation for Vietnamese + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + + s = "!#%&'()*+,-./:;<=>?@^_`{|}~′″°" + + punct = pynini.union(*s) + self.punct_marks = punct + self.graph = punct + + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/vi/taggers/range.py b/nemo_text_processing/text_normalization/vi/taggers/range.py new file mode 100644 index 000000000..e39be6a3b --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/range.py @@ -0,0 +1,66 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_SPACE, GraphFst + + +class RangeFst(GraphFst): + """ + Finite state transducer for classifying Vietnamese ranges with dash "-" + Examples: + 10k-20k -> tokens { name: "mười nghìn đến hai mười nghìn" } + 10h-8h -> tokens { name: "mười giờ đến tám giờ" } + 10$-20$ -> tokens { name: "mười đô la đến hai mười đô la" } + + Args: + time: composed time tagger and verbalizer + date: composed date tagger and verbalizer + decimal: composed decimal tagger and verbalizer + money: composed money tagger and verbalizer + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__( + self, + time: GraphFst, + date: GraphFst, + decimal: GraphFst, + money: GraphFst, + measure: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="range", kind="classify", deterministic=deterministic) + + delete_space = pynini.closure(pynutil.delete(NEMO_SPACE), 0, 1) + + # Pattern: X-Y -> X đến Y + # This will handle time ranges, date ranges, decimal ranges, and money ranges with dash + range_pattern = ( + (time | date | decimal | money | measure) + + delete_space + + pynini.cross("-", " đến ") + + delete_space + + (time | date | decimal | money | measure) + ) + + self.graph = range_pattern + + # Convert to final FST format + self.graph = self.graph.optimize() + graph = pynutil.insert("name: \"") + self.graph + pynutil.insert("\"") + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/vi/taggers/roman.py b/nemo_text_processing/text_normalization/vi/taggers/roman.py new file mode 100644 index 000000000..2f8b043fa --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/roman.py @@ -0,0 +1,91 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying roman numbers in Vietnamese context: + e.g. "thế kỉ XV" -> tokens { roman { key_cardinal: "thế kỉ" integer: "mười lăm" } } + e.g. "thế kỷ IV" -> tokens { roman { key_cardinal: "thế kỷ" integer: "bốn" } } + e.g. "thứ IV" -> tokens { roman { key_cardinal: "thứ" integer: "bốn" } } + e.g. "chương III" -> tokens { roman { key_cardinal: "chương" integer: "ba" } } + e.g. "phần ix" -> tokens { roman { key_cardinal: "phần" integer: "chín" } } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + key_words = [] + key_word_path = get_abs_path("data/roman/key_word.tsv") + for k_word in load_labels(key_word_path): + key_words.append(k_word[0]) + + key_words_fst = pynini.union(*[pynini.accep(word) for word in key_words]).optimize() + + roman_numeral_path = get_abs_path("data/roman/roman_numerals.tsv") + roman_numeral_pairs = load_labels(roman_numeral_path) + + roman_to_arabic = {} + for roman, value in roman_numeral_pairs: + roman_to_arabic[roman] = value + roman_to_arabic[roman.lower()] = value + + self.arabic_to_roman = {} + for roman, value in roman_numeral_pairs: + self.arabic_to_roman[int(value)] = roman + + valid_roman_pairs = [] + for i in range(1, 4000): + roman_upper = self._int_to_roman(i) + roman_lower = roman_upper.lower() + valid_roman_pairs.append((roman_upper, str(i))) + valid_roman_pairs.append((roman_lower, str(i))) + + roman_to_arabic_fst = pynini.string_map(valid_roman_pairs).optimize() + + cardinal_graph = cardinal.graph + + graph = ( + pynutil.insert("key_cardinal: \"") + + key_words_fst + + pynutil.insert("\"") + + pynini.accep(NEMO_SPACE) + + pynutil.insert("integer: \"") + + pynini.compose(roman_to_arabic_fst, cardinal_graph) + + pynutil.insert("\"") + ).optimize() + + self.fst = self.add_tokens(graph).optimize() + + def _int_to_roman(self, num): + values = sorted(self.arabic_to_roman.keys(), reverse=True) + + roman_num = '' + for value in values: + while num >= value: + roman_num += self.arabic_to_roman[value] + num -= value + + return roman_num diff --git a/nemo_text_processing/text_normalization/vi/taggers/time.py b/nemo_text_processing/text_normalization/vi/taggers/time.py new file mode 100644 index 000000000..aebcae1b3 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/time.py @@ -0,0 +1,140 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_DIGIT, + NEMO_SPACE, + GraphFst, + convert_space, + insert_space, +) +from nemo_text_processing.text_normalization.vi.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for classifying time in Vietnamese. + + Supports various formats including: + - Digital formats: "8:30", "14:45", "5:20:35" + - Vietnamese formats: "14 giờ 30 phút", "2 giờ 15 phút 10 giây" + - Abbreviated formats: "9h", "9g", "14h30", "14g30", "3p20s" + - With time zones: "8:23 gmt", "15h cst" + + Args: + cardinal: CardinalFst for number conversion + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="time", kind="classify", deterministic=deterministic) + + time_zone = pynini.string_file(get_abs_path("data/time/time_zones.tsv")) + delete_leading_zero = (pynutil.delete("0").ques | (NEMO_DIGIT - "0")) + NEMO_DIGIT + cardinal_graph = cardinal.graph + + hours = pynini.union(*[str(x) for x in range(0, 25)]) + minutes_seconds = pynini.union(*[str(x) for x in range(0, 60)]) + + def label(name, graph): + return pynutil.insert(f'{name}: "') + graph + pynutil.insert('"') + + hour = label('hours', delete_leading_zero @ hours @ cardinal_graph) + minute = label('minutes', delete_leading_zero @ minutes_seconds @ cardinal_graph) + second = label('seconds', delete_leading_zero @ minutes_seconds @ cardinal_graph) + zone = label('zone', convert_space(time_zone)) + + h_suffix = pynini.union(pynutil.delete("h"), pynutil.delete("g")) + h_word = pynutil.delete(" giờ") + m_word = pynutil.delete(" phút") + s_word = pynutil.delete(" giây") + + opt_zone_space = pynini.closure(pynini.accep(NEMO_SPACE) + zone, 0, 1) + opt_zone = pynini.closure(zone, 0, 1) + preserve = pynutil.insert(" preserve_order: true") + + # Define sub-patterns for better readability + # Digital formats + pattern_hour_minute = hour + pynutil.delete(":") + insert_space + minute + opt_zone_space + + pattern_hour_minute_second = ( + hour + + pynutil.delete(":") + + insert_space + + minute + + pynutil.delete(":") + + insert_space + + second + + opt_zone_space + + preserve + ) + + # Abbreviated formats + pattern_hour_suffix = hour + h_suffix + opt_zone_space + pattern_hour_suffix_minute = hour + h_suffix + minute + opt_zone + pattern_minute_p = minute + pynutil.delete("p") + pattern_second_s = second + pynutil.delete("s") + pattern_minute_p_second_s = minute + pynutil.delete("p") + insert_space + second + pynutil.delete("s") + + # Vietnamese word formats + pattern_hour_word = hour + h_word + opt_zone_space + + pattern_hour_word_minute = hour + h_word + pynutil.delete(NEMO_SPACE) + minute + m_word + opt_zone_space + + pattern_hour_word_minute_second = ( + hour + + h_word + + pynutil.delete(NEMO_SPACE) + + minute + + m_word + + pynutil.delete(NEMO_SPACE) + + second + + s_word + + opt_zone_space + + preserve + ) + + pattern_minute_word = minute + m_word + pattern_minute_word_second = minute + m_word + pynutil.delete(NEMO_SPACE) + second + s_word + pattern_second_word = second + s_word + + # Time zone specific patterns + pattern_hour_suffix_space_zone = hour + h_suffix + pynini.accep(NEMO_SPACE) + zone + pattern_hour_suffix_zone = hour + h_suffix + zone + + patterns = [ + pattern_hour_minute, + pattern_hour_minute_second, + pattern_hour_suffix, + pattern_hour_suffix_minute, + pattern_minute_p, + pattern_second_s, + pattern_minute_p_second_s, + pattern_hour_word, + pattern_hour_word_minute, + pattern_hour_word_minute_second, + pattern_minute_word, + pattern_minute_word_second, + pattern_second_word, + pattern_hour_suffix_space_zone, + pattern_hour_suffix_zone, + ] + + final_graph = pynini.union(*patterns).optimize() + + self.fst = self.add_tokens(final_graph).optimize() diff --git a/nemo_text_processing/text_normalization/vi/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/vi/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..311741cde --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/tokenize_and_classify.py @@ -0,0 +1,170 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.vi.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.vi.taggers.date import DateFst +from nemo_text_processing.text_normalization.vi.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.vi.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.vi.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.vi.taggers.money import MoneyFst +from nemo_text_processing.text_normalization.vi.taggers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.vi.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.vi.taggers.range import RangeFst +from nemo_text_processing.text_normalization.vi.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.vi.taggers.time import TimeFst +from nemo_text_processing.text_normalization.vi.taggers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.vi.taggers.word import WordFst +from nemo_text_processing.text_normalization.vi.verbalizers.cardinal import CardinalFst as VCardinalFst +from nemo_text_processing.text_normalization.vi.verbalizers.date import DateFst as VDateFst +from nemo_text_processing.text_normalization.vi.verbalizers.decimal import DecimalFst as VDecimalFst +from nemo_text_processing.text_normalization.vi.verbalizers.fraction import FractionFst as VFractionFst +from nemo_text_processing.text_normalization.vi.verbalizers.measure import MeasureFst as VMeasureFst +from nemo_text_processing.text_normalization.vi.verbalizers.money import MoneyFst as VMoneyFst +from nemo_text_processing.text_normalization.vi.verbalizers.time import TimeFst as VTimeFst +from nemo_text_processing.utils.logging import logger + + +class ClassifyFst(GraphFst): + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join( + cache_dir, + f"vi_tn_{deterministic}_deterministic_{input_case}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logger.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logger.info(f"Creating Vietnamese ClassifyFst grammars.") + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + + whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic) + whitelist_graph = whitelist.fst + + word_graph = WordFst(deterministic=deterministic).fst + + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + fraction_graph = fraction.fst + + date = DateFst(cardinal=cardinal, deterministic=deterministic) + date_graph = date.fst + + roman = RomanFst(cardinal=cardinal, deterministic=deterministic) + roman_graph = roman.fst + + time_fst = TimeFst(cardinal=cardinal, deterministic=deterministic) + time_graph = time_fst.fst + + money = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic) + money_graph = money.fst + + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + + # Create composed verbalizers for range processing + v_cardinal = VCardinalFst(deterministic=deterministic) + v_date = VDateFst(deterministic=deterministic) + date_final = pynini.compose(date_graph, v_date.fst) + + v_decimal = VDecimalFst(v_cardinal, deterministic=deterministic) + decimal_final = pynini.compose(decimal_graph, v_decimal.fst) + + v_time = VTimeFst(deterministic=deterministic) + time_final = pynini.compose(time_graph, v_time.fst) + + v_money = VMoneyFst(deterministic=deterministic) + money_final = pynini.compose(money_graph, v_money.fst) + + v_fraction = VFractionFst(deterministic=deterministic) + v_measure = VMeasureFst( + decimal=v_decimal, cardinal=v_cardinal, fraction=v_fraction, deterministic=deterministic + ) + measure_final = pynini.compose(measure_graph, v_measure.fst) + + # Create range graph + range_fst = RangeFst( + time=time_final, + date=date_final, + decimal=decimal_final, + money=money_final, + measure=measure_final, + deterministic=deterministic, + ) + range_graph = range_fst.fst + + classify = ( + pynutil.add_weight(whitelist_graph, 1.01) + | pynutil.add_weight(money_graph, 1.1) + | pynutil.add_weight(range_graph, 1.1) + | pynutil.add_weight(decimal_graph, 1.1) + | pynutil.add_weight(date_graph, 1.1) + | pynutil.add_weight(cardinal_graph, 1.1) + | pynutil.add_weight(ordinal_graph, 1.1) + | pynutil.add_weight(fraction_graph, 1.1) + | pynutil.add_weight(time_graph, 1.1) + | pynutil.add_weight(measure_graph, 1.1) + | pynutil.add_weight(word_graph, 100) + | pynutil.add_weight(roman_graph, 101) + ) + punct = ( + pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, 2.1) + pynutil.insert(" }") + ) # Lower priority than semantic classes + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure((delete_extra_space).ques + token_plus_punct) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) diff --git a/nemo_text_processing/text_normalization/vi/taggers/whitelist.py b/nemo_text_processing/text_normalization/vi/taggers/whitelist.py new file mode 100644 index 000000000..d2775f205 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/whitelist.py @@ -0,0 +1,70 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import GraphFst, convert_space +from nemo_text_processing.text_normalization.vi.utils import get_abs_path, load_labels + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for classifying whitelist for Vietnamese, e.g. + "h" -> tokens { name: "giờ" } + "p" -> tokens { name: "phút" } + "s" -> tokens { name: "giây" } + This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv". + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + input_file: path to a file with whitelist replacements + """ + + def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + super().__init__(name="whitelist", kind="classify", deterministic=deterministic) + + def _get_whitelist_graph(input_case, file): + whitelist = load_labels(file) + if input_case == "lower_cased": + whitelist = [[x[0].lower()] + x[1:] for x in whitelist] + graph = pynini.string_map(whitelist) + return graph + + graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) + if not deterministic and input_case != "lower_cased": + graph |= pynutil.add_weight( + _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 + ) + + if input_file: + whitelist_provided = _get_whitelist_graph(input_case, input_file) + if not deterministic: + graph |= whitelist_provided + else: + graph = whitelist_provided + + # Add time units from time_units.tsv for better time handling + if not deterministic: + time_units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/time/time_units.tsv")) + graph |= time_units_graph + + self.graph = graph + self.final_graph = convert_space(self.graph).optimize() + self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() + + # Add tokens wrapper + self.fst = self.add_tokens(self.fst) diff --git a/nemo_text_processing/text_normalization/vi/taggers/word.py b/nemo_text_processing/text_normalization/vi/taggers/word.py new file mode 100644 index 000000000..96d203467 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/taggers/word.py @@ -0,0 +1,46 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_ALPHA, NEMO_DIGIT, NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Vietnamese words. + e.g. ngày -> name: "ngày" + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + + # Symbols that should cause token breaks + # Include measure symbols, currency symbols, and digits + symbols_to_exclude = pynini.union("°", "′", "″", "$", "€", "₩", "£", "¥", "#", "%", "₫", NEMO_DIGIT).optimize() + + word_chars = pynini.closure(pynini.difference(NEMO_NOT_SPACE, symbols_to_exclude), 1) + default_word_graph = word_chars + + alpha_word_graph = pynini.closure(NEMO_ALPHA, 1) + + graph = pynutil.add_weight(alpha_word_graph, -1.0) | default_word_graph + + word = pynutil.insert("name: \"") + graph + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/vi/utils.py b/nemo_text_processing/text_normalization/vi/utils.py new file mode 100644 index 000000000..1f23ee3cc --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/utils.py @@ -0,0 +1,42 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/__init__.py b/nemo_text_processing/text_normalization/vi/verbalizers/__init__.py new file mode 100644 index 000000000..b2de1dca7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/vi/verbalizers/cardinal.py new file mode 100644 index 000000000..b096e759d --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/cardinal.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese cardinal numbers, e.g. + cardinal { negative: "true" integer: "hai mươi ba" } -> âm hai mươi ba + cardinal { integer: "một trăm" } -> một trăm + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + # Handle negative sign - Vietnamese uses "âm" for negative numbers + self.optional_sign = pynini.cross("negative: \"true\"", "âm ") + if not deterministic: + # Alternative ways to say negative in Vietnamese + self.optional_sign |= pynini.cross("negative: \"true\"", "trừ ") + self.optional_sign |= pynini.cross("negative: \"true\"", "âm ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + # Handle the integer part + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + # Combine negative sign with integer + self.numbers = self.optional_sign + integer + + # Delete the token structure and create final FST + delete_tokens = self.delete_tokens(self.numbers) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/date.py b/nemo_text_processing/text_normalization/vi/verbalizers/date.py new file mode 100644 index 000000000..4e918e3d4 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/date.py @@ -0,0 +1,71 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space, insert_space + + +class DateFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese dates, e.g. + date { day: "mười lăm" month: "một" year: "hai nghìn hai mươi tư" } + -> ngày mười lăm tháng một năm hai nghìn hai mươi tư + + date { month: "tư" year: "hai nghìn hai mươi tư" } + -> tháng tư năm hai nghìn hai mươi tư + + date { year: "hai mươi" era: "sau công nguyên" } + -> năm hai mươi sau công nguyên + + date { ordinal: "mười" era: "trước công nguyên" } + -> năm thứ mười trước công nguyên + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="date", kind="verbalize", deterministic=deterministic) + + quoted_content = pynini.closure(NEMO_NOT_QUOTE) + + day_expr = pynutil.delete("day: \"") + quoted_content + pynutil.delete("\"") + day_with_prefix = pynutil.insert("ngày ") + day_expr + + month_expr = pynutil.delete("month: \"") + quoted_content + pynutil.delete("\"") + month_with_prefix = pynutil.insert("tháng ") + month_expr + + year_expr = pynutil.delete("year: \"") + quoted_content + pynutil.delete("\"") + year_with_prefix = pynutil.insert("năm ") + year_expr + + era_expr = pynutil.delete("era: \"") + quoted_content + pynutil.delete("\"") + + ordinal_expr = pynutil.delete("ordinal: \"") + quoted_content + pynutil.delete("\"") + ordinal_with_prefix = pynutil.insert("năm thứ ") + ordinal_expr + + date_graph = pynini.union( + day_with_prefix + + delete_space + + insert_space + + month_with_prefix + + delete_space + + insert_space + + year_with_prefix, + month_with_prefix + delete_space + insert_space + year_with_prefix, + day_with_prefix + delete_space + insert_space + month_with_prefix, + year_with_prefix, + year_with_prefix + delete_space + insert_space + era_expr, + ordinal_with_prefix + delete_space + insert_space + era_expr, + ) + + self.fst = self.delete_tokens(date_graph).optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/decimal.py b/nemo_text_processing/text_normalization/vi/verbalizers/decimal.py new file mode 100644 index 000000000..c94dd0653 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/decimal.py @@ -0,0 +1,103 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_COMMA_VI, + NEMO_NOT_QUOTE, + GraphFst, + delete_space, + insert_space, +) + + +class DecimalFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese decimal numbers, e.g. + decimal { negative: "true" integer_part: "mười hai" fractional_part: "năm" quantity: "tỷ" } -> âm mười hai phẩy năm tỷ + decimal { integer_part: "tám trăm mười tám" fractional_part: "ba không ba" } -> tám trăm mười tám phẩy ba không ba + decimal { integer_part: "không" fractional_part: "hai" quantity: "triệu" } -> không phẩy hai triệu + + Args: + cardinal: CardinalFst instance for handling integer verbalization + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, cardinal, deterministic: bool = True): + super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) + + # Basic components + integer = pynutil.delete("integer_part:") + cardinal.integer + fractional = ( + pynutil.delete("fractional_part:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + quantity = ( + pynutil.delete("quantity:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + # Negative handling + negative = pynini.cross("negative: \"true\"", "âm ") + if not deterministic: + negative |= pynini.cross("negative: \"true\"", "trừ ") + optional_negative = pynini.closure(negative + delete_space, 0, 1) + + # Simple patterns + simple_integer = integer + + integer_with_quantity = integer + delete_space + insert_space + quantity + + decimal_with_comma = ( + integer + delete_space + insert_space + pynutil.insert(NEMO_COMMA_VI) + insert_space + fractional + ) + + decimal_with_quantity = ( + integer + + delete_space + + insert_space + + pynutil.insert(NEMO_COMMA_VI) + + insert_space + + fractional + + delete_space + + insert_space + + quantity + ) + + fractional_only = ( + pynini.closure(integer + delete_space + insert_space, 0, 1) + + pynutil.insert(NEMO_COMMA_VI) + + insert_space + + fractional + ) + + # Group all patterns + all_patterns = pynini.union( + simple_integer, integer_with_quantity, decimal_with_comma, decimal_with_quantity, fractional_only + ) + + # Combine with negative handling + graph = optional_negative + all_patterns + + self.numbers = graph + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/fraction.py b/nemo_text_processing/text_normalization/vi/verbalizers/fraction.py new file mode 100644 index 000000000..675d959df --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/fraction.py @@ -0,0 +1,55 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class FractionFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese fraction numbers, e.g. + fraction { negative: "true" integer_part: "hai mươi ba" numerator: "một" denominator: "năm" } -> âm hai mươi ba và một phần năm + fraction { numerator: "ba" denominator: "chín" } -> ba phần chín + fraction { integer_part: "một trăm" numerator: "hai" denominator: "ba" } -> một trăm và hai phần ba + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + + optional_sign = pynini.cross("negative: \"true\"", "âm ") + if not deterministic: + optional_sign |= pynini.cross("negative: \"true\"", "trừ ") + optional_sign = pynini.closure(optional_sign + delete_space, 0, 1) + + part = pynini.closure(NEMO_NOT_QUOTE) + delete_quotes = delete_space + pynutil.delete("\"") + part + pynutil.delete("\"") + + integer_tagged = pynutil.delete("integer_part:") + delete_quotes + numerator_tagged = pynutil.delete("numerator:") + delete_quotes + denominator_tagged = pynutil.delete("denominator:") + delete_quotes + + fraction_part = numerator_tagged + delete_space + pynutil.insert(" phần ") + denominator_tagged + + simple_fraction = fraction_part + mixed_fraction = integer_tagged + delete_space + pynutil.insert(" và ") + fraction_part + + self.numbers = optional_sign + (simple_fraction | mixed_fraction) + + self.fst = self.delete_tokens(self.numbers).optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/measure.py b/nemo_text_processing/text_normalization/vi/verbalizers/measure.py new file mode 100644 index 000000000..49283eb6f --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/measure.py @@ -0,0 +1,67 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + GraphFst, + delete_preserve_order, + delete_space, + extract_field, + extract_wrapper_content, + insert_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing measure for Vietnamese, e.g. + measure { negative: "true" cardinal { integer: "mười hai" } units: "ki lô gam" } -> âm mười hai ki lô gam + measure { decimal { integer_part: "mười hai" fractional_part: "năm" } units: "ki lô gam" } -> mười hai phẩy năm ki lô gam + measure { cardinal { integer: "một" } units: "ki lô gam" } -> một ki lô gam + + Args: + decimal: DecimalFst verbalizer + cardinal: CardinalFst verbalizer + fraction: FractionFst verbalizer + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool = True): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + # Extract components + unit = extract_field("units") + + # Handle negative sign - Vietnamese uses "âm" for negative numbers + optional_negative = pynini.closure(pynini.cross("negative: \"true\"", "âm ") + delete_space, 0, 1) + if not deterministic: + # Alternative ways to say negative in Vietnamese + optional_negative |= pynini.closure(pynini.cross("negative: \"true\"", "trừ ") + delete_space, 0, 1) + + # Combine all number types into single graph + number_graph = ( + extract_wrapper_content("decimal", decimal.numbers) + | extract_wrapper_content("cardinal", cardinal.numbers) + | extract_wrapper_content("fraction", fraction.numbers) + ) + + # Main pattern: [negative] number + space + unit (most common case) + graph = optional_negative + number_graph + delete_space + insert_space + unit + + # Handle preserve_order: [negative] unit + space + number + graph |= optional_negative + unit + delete_space + insert_space + number_graph + delete_preserve_order + + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/money.py b/nemo_text_processing/text_normalization/vi/verbalizers/money.py new file mode 100644 index 000000000..139ea8a1d --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/money.py @@ -0,0 +1,143 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_COMMA_VI, + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.vi.utils import get_abs_path + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing money, e.g. + money { integer_part: "mười" currency_maj: "đồng" } -> "mười đồng" + money { integer_part: "mười" quantity: "triệu" currency_maj: "đồng" } -> "mười triệu đồng" + money { integer_part: "mười" currency_maj: "đô la" fractional_part: "năm mươi" currency_min: "xu" preserve_order: true } -> "mười đô la năm mươi xu" + money { fractional_part: "năm mươi" currency_min: "xu" preserve_order: true } -> "năm mươi xu" + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + integer_part = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + fractional_part = ( + pynutil.delete('fractional_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ) + quantity = pynutil.delete('quantity: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + currency_maj = pynutil.delete('currency_maj: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + currency_min = pynutil.delete('currency_min: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + + # Following English prioritization pattern for better determinism + + # 1. Minor only: fractional + minor (highest priority for fractional-only cases) + graph_minor = fractional_part + delete_space + insert_space + currency_min + delete_preserve_order + + # 2. Major + minor: integer + major + fractional + minor (for complete cases like 10,5$) + graph_integer_with_minor = ( + integer_part + + delete_space + + insert_space + + currency_maj + + delete_space + + insert_space + + fractional_part + + delete_space + + insert_space + + currency_min + + delete_preserve_order + ) + + # 3. Simple integer + currency (most common case) + graph_integer = integer_part + delete_space + insert_space + currency_maj + + # 4. With quantity: integer + quantity + currency + graph_with_quantity = ( + integer_part + delete_space + insert_space + quantity + delete_space + insert_space + currency_maj + ) + + # 5. Decimal format (using "phẩy" for comma) - for cases like 10,5 đồng + graph_decimal = ( + integer_part + + delete_space + + insert_space + + pynutil.insert(NEMO_COMMA_VI) + + insert_space + + fractional_part + + delete_space + + insert_space + + currency_maj + ) + + # 6. Decimal with quantity: integer + fractional + quantity + currency - for cases like 2,5 triệu đồng + graph_decimal_with_quantity = ( + integer_part + + delete_space + + insert_space + + pynutil.insert(NEMO_COMMA_VI) + + insert_space + + fractional_part + + delete_space + + insert_space + + quantity + + delete_space + + insert_space + + currency_maj + ) + + # Create main graph with proper priority order (similar to English) + graph = ( + graph_minor # Handle minor-only cases first + | graph_integer_with_minor # Handle major+minor cases + | graph_decimal_with_quantity # Handle decimal with quantity cases (before simpler decimal) + | graph_with_quantity # Handle quantity cases + | graph_decimal # Handle decimal cases + | graph_integer # Handle simple cases (most common, lowest priority) + ) + + per_units_non_metric = pynini.string_file(get_abs_path("data/money/per_unit_non_metric.tsv")) + + per_unit_prefixes = pynini.string_file(get_abs_path("data/money/per_unit_prefixes.tsv")) + per_unit_bases = pynini.string_file(get_abs_path("data/money/per_unit_bases.tsv")) + + prefixes_vn = pynini.project(per_unit_prefixes, "output") + bases_vn = pynini.project(per_unit_bases, "output") + + one = pynini.accep("một") + + # Accept metric combinations: "một ki lô gam" + metric_per_units = one + insert_space + prefixes_vn + insert_space + bases_vn + standalone_per_units = one + insert_space + bases_vn + + # Combine all per_unit recognitions + per_units = per_units_non_metric | metric_per_units | standalone_per_units + per_units_normalized = pynini.project(per_units, "output") + per_unit_pattern = ( + pynutil.delete(' morphosyntactic_features: "') + insert_space + per_units_normalized + pynutil.delete('"') + ) + + # Optional per-unit suffix + graph += per_unit_pattern.ques + + # Handle preserve_order deletion (should be last) + graph += (delete_space + pynutil.delete("preserve_order: true")).ques + + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/vi/verbalizers/ordinal.py new file mode 100644 index 000000000..0a0bf3ac0 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/ordinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class OrdinalFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese ordinals, e.g. + ordinal { integer: "nhất" } -> thứ nhất + ordinal { integer: "tư" } -> thứ tư + ordinal { integer: "mười lăm" } -> thứ mười lăm + ordinal { integer: "một trăm" } -> thứ một trăm + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) + + quoted_content = pynini.closure(NEMO_NOT_QUOTE) + + integer = ( + pynutil.delete("integer:") + delete_space + pynutil.delete("\"") + quoted_content + pynutil.delete("\"") + ) + + ordinal_pattern = pynutil.insert("thứ ") + integer + + self.ordinal_graph = ordinal_pattern + + delete_tokens = self.delete_tokens(self.ordinal_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/vi/verbalizers/post_processing.py new file mode 100644 index 000000000..9b56fb98c --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/post_processing.py @@ -0,0 +1,139 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Dict, List + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_SIGMA, NEMO_SPACE, generator_main +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processes an entire Vietnamese sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( một trăm hai mươi ba ) " -> "(một trăm hai mươi ba)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "vi_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def get_vietnamese_punct_config(self) -> Dict[str, List[str]]: + """ + Returns Vietnamese-specific punctuation configuration. + This method can be easily modified or extended for different Vietnamese punctuation rules. + """ + return { + # Punctuation that should not have space before them + 'no_space_before': [",", ".", "!", "?", ":", ";", ")", r"\]", "}"], + # Punctuation that should not have space after them + 'no_space_after': ["(", r"\[", "{"], + # Punctuation that can have space before them (exceptions) + 'allow_space_before': ["&", "-", "—", "–", "(", r"\[", "{", "\"", "'", "«", "»"], + # Special Vietnamese punctuation handling + 'vietnamese_special': { + # Vietnamese quotation marks + 'quotes': ["\"", "'", "«", "»", """, """, "'", "'"], + # Vietnamese dashes and separators + 'dashes': ["-", "—", "–"], + # Vietnamese brackets + 'brackets': ["(", ")", r"\[", r"\]", "{", "}"], + }, + } + + def set_punct_dict(self): + # Vietnamese punctuation marks that might need special handling + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '`', + '´', + '’', + '‛', + '′', + '‵', + 'ꞌ', + '＇', + '｀', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks for Vietnamese. + + Uses dynamic configuration for flexible punctuation handling. + Vietnamese punctuation spacing rules are defined in get_vietnamese_punct_config(). + """ + # Get dynamic punctuation configuration + punct_config = self.get_vietnamese_punct_config() + + # Extract configuration + no_space_before_punct = punct_config['no_space_before'] + no_space_after_punct = punct_config['no_space_after'] + + # Create FSTs for punctuation rules + no_space_before_punct_fst = pynini.union(*no_space_before_punct) + no_space_after_punct_fst = pynini.union(*no_space_after_punct) + + delete_space = pynutil.delete(NEMO_SPACE) + + # Rule 1: Remove space before punctuation (primary rule) + remove_space_before = pynini.cdrewrite( + delete_space + no_space_before_punct_fst, # " ," -> "," + "", # any context before + "", # any context after + NEMO_SIGMA, + ).optimize() + + # Rule 2: Remove space after opening brackets + remove_space_after = pynini.cdrewrite( + no_space_after_punct_fst + delete_space, "", "", NEMO_SIGMA # "( " -> "(" + ).optimize() + + # Combine the two main rules + graph = pynini.compose(remove_space_before, remove_space_after) + + return graph.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/range.py b/nemo_text_processing/text_normalization/vi/verbalizers/range.py new file mode 100644 index 000000000..ad9235ac2 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/range.py @@ -0,0 +1,46 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, +) + + +class RangeFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese ranges. + Range tokens are already verbalized by the tagger, so this just extracts the content. + e.g. tokens { name: "mười nghìn đến hai mười nghìn" } -> mười nghìn đến hai mười nghìn + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="range", kind="verbalize", deterministic=deterministic) + + # Range content is already verbalized by the tagger, just extract it + chars = pynini.closure(NEMO_CHAR - NEMO_SPACE, 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", NEMO_SPACE), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/roman.py b/nemo_text_processing/text_normalization/vi/verbalizers/roman.py new file mode 100644 index 000000000..3c3097f6a --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/roman.py @@ -0,0 +1,45 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + + +class RomanFst(GraphFst): + """ + Finite state transducer for verbalizing Roman numerals in Vietnamese + e.g. tokens { roman { key_cardinal: "thế kỉ" integer: "mười lăm" } } -> thế kỉ mười lăm + e.g. tokens { roman { key_cardinal: "thế kỷ" integer: "bốn" } } -> thế kỷ bốn + e.g. tokens { roman { key_cardinal: "thứ" integer: "bốn" } } -> thứ bốn + e.g. tokens { roman { integer: "mười lăm" } } -> mười lăm + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="roman", kind="verbalize", deterministic=deterministic) + + key_cardinal = pynutil.delete("key_cardinal: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + integer = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + + graph_with_key = key_cardinal + delete_space + pynutil.insert(NEMO_SPACE) + integer + graph_without_key = integer + graph = pynini.union(graph_with_key, graph_without_key) + delete_tokens = self.delete_tokens(graph) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/time.py b/nemo_text_processing/text_normalization/vi/verbalizers/time.py new file mode 100644 index 000000000..966b8bad1 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/time.py @@ -0,0 +1,174 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SPACE, + GraphFst, + convert_space, + delete_preserve_order, + delete_space, + extract_field, +) +from nemo_text_processing.text_normalization.vi.utils import get_abs_path + + +class TimeFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese time. + + Converts tagged time entities into spoken form, e.g.: + - time { hours: "tám" minutes: "ba mươi" } -> tám giờ ba mươi phút + - time { hours: "mười bốn" minutes: "mười lăm" } -> mười bốn giờ mười lăm phút + - time { hours: "chín" } -> chín giờ + - time { minutes: "ba" seconds: "hai mươi" } -> ba phút hai mươi giây + - time { hours: "tám" minutes: "hai mươi ba" zone: "g m t" } -> tám giờ hai mươi ba phút GMT + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="time", kind="verbalize", deterministic=deterministic) + + time_zone = convert_space(pynini.string_file(get_abs_path("data/time/time_zones.tsv"))) + + # Extract components + hour_component = extract_field("hours") + timezone_component = extract_field("zone") @ time_zone + + # Handle zero and non-zero components + zero_minute_component = pynutil.delete("minutes:") + delete_space + pynutil.delete("\"không\"") + zero_second_component = pynutil.delete("seconds:") + delete_space + pynutil.delete("\"không\"") + + non_zero_minute_component = ( + pynutil.delete("minutes:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE - pynini.accep("không")) + + pynutil.delete("\"") + ) + non_zero_second_component = ( + pynutil.delete("seconds:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE - pynini.accep("không")) + + pynutil.delete("\"") + ) + + # Components with units + hour_with_unit = hour_component + pynutil.insert(" giờ") + minute_with_unit = non_zero_minute_component + pynutil.insert(" phút") + second_with_unit = non_zero_second_component + pynutil.insert(" giây") + + # Optional components + optional_timezone = pynini.closure(delete_space + pynutil.insert(NEMO_SPACE) + timezone_component, 0, 1) + optional_preserve_order = pynini.closure(delete_space + delete_preserve_order, 0, 1) + + # Pattern 1: hours + optional zero minutes/seconds + optional timezone + pattern_hours_only = ( + hour_with_unit + + pynini.closure(delete_space + zero_minute_component, 0, 1) + + pynini.closure(delete_space + zero_second_component, 0, 1) + + optional_timezone + + optional_preserve_order + ) + + # Pattern 2: hours + minutes + optional zero seconds + optional timezone + pattern_hours_minutes = ( + hour_with_unit + + delete_space + + pynutil.insert(NEMO_SPACE) + + minute_with_unit + + pynini.closure(delete_space + zero_second_component, 0, 1) + + optional_timezone + + optional_preserve_order + ) + + # Pattern 3: hours + zero minutes + seconds + optional timezone + pattern_hours_seconds = ( + hour_with_unit + + delete_space + + zero_minute_component + + delete_space + + pynutil.insert(NEMO_SPACE) + + second_with_unit + + optional_timezone + + optional_preserve_order + ) + + # Pattern 4: hours + minutes + seconds + optional timezone + pattern_hours_minutes_seconds = ( + hour_with_unit + + delete_space + + pynutil.insert(NEMO_SPACE) + + minute_with_unit + + delete_space + + pynutil.insert(NEMO_SPACE) + + second_with_unit + + optional_timezone + + optional_preserve_order + ) + + # Pattern 5: minutes only + optional zero seconds + pattern_minutes_only = minute_with_unit + pynini.closure(delete_space + zero_second_component, 0, 1) + + # Pattern 6: minutes + seconds + pattern_minutes_seconds = minute_with_unit + delete_space + pynutil.insert(NEMO_SPACE) + second_with_unit + + # Pattern 7: seconds only + pattern_seconds_only = second_with_unit + + patterns = [ + pattern_hours_only, + pattern_hours_minutes, + pattern_hours_seconds, + pattern_hours_minutes_seconds, + pattern_minutes_only, + pattern_minutes_seconds, + pattern_seconds_only, + ] + + final_graph = pynini.union(*patterns) + + if not deterministic: + # Add special case for half hour ("rưỡi") + half_hour = ( + pynutil.delete("minutes:") + delete_space + pynutil.delete("\"ba mươi\"") + pynutil.insert("rưỡi") + ) + half_hour_pattern = ( + hour_with_unit + + delete_space + + pynutil.insert(NEMO_SPACE) + + half_hour + + optional_timezone + + optional_preserve_order + ) + self.graph = pynini.union(final_graph, half_hour_pattern) + else: + self.graph = final_graph + + # Remove zero minutes and seconds from output + remove_zero_minutes = pynini.cdrewrite(pynutil.delete(" không phút"), "", "", pynini.closure(NEMO_NOT_QUOTE)) + remove_zero_seconds = pynini.cdrewrite(pynutil.delete(" không giây"), "", "", pynini.closure(NEMO_NOT_QUOTE)) + + self.fst = ( + self.delete_tokens(self.graph + optional_preserve_order).optimize() + @ remove_zero_minutes + @ remove_zero_seconds + ) diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize.py new file mode 100644 index 000000000..30008275e --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize.py @@ -0,0 +1,85 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.vi.graph_utils import GraphFst +from nemo_text_processing.text_normalization.vi.verbalizers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.vi.verbalizers.date import DateFst +from nemo_text_processing.text_normalization.vi.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.vi.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.vi.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.vi.verbalizers.money import MoneyFst +from nemo_text_processing.text_normalization.vi.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.vi.verbalizers.range import RangeFst +from nemo_text_processing.text_normalization.vi.verbalizers.roman import RomanFst +from nemo_text_processing.text_normalization.vi.verbalizers.time import TimeFst +from nemo_text_processing.text_normalization.vi.verbalizers.whitelist import WhiteListFst +from nemo_text_processing.text_normalization.vi.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + whitelist = WhiteListFst(deterministic=deterministic) + whitelist_graph = whitelist.fst + + word = WordFst(deterministic=deterministic) + word_graph = word.fst + + ordinal = OrdinalFst(deterministic=deterministic) + ordinal_graph = ordinal.fst + + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + decimal_graph = decimal.fst + + fraction = FractionFst(deterministic=deterministic) + fraction_graph = fraction.fst + + date = DateFst(deterministic=deterministic) + date_graph = date.fst + + roman = RomanFst(deterministic=deterministic) + roman_graph = roman.fst + + time_fst = TimeFst(deterministic=deterministic) + time_graph = time_fst.fst + + money = MoneyFst(deterministic=deterministic) + money_graph = money.fst + + measure = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction, deterministic=deterministic) + measure_graph = measure.fst + + range_fst = RangeFst(deterministic=deterministic) + range_graph = range_fst.fst + + graph = ( + cardinal_graph + | whitelist_graph + | word_graph + | ordinal_graph + | decimal_graph + | fraction_graph + | date_graph + | roman_graph + | time_graph + | money_graph + | measure_graph + | range_graph + ) + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize_final.py new file mode 100644 index 000000000..aa8344459 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/verbalize_final.py @@ -0,0 +1,72 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.vi.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.vi.verbalizers.word import WordFst +from nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire Vietnamese sentence, e.g. + tokens { name: "xin" } tokens { cardinal { integer: "một trăm" } } tokens { name: "chào" } + -> xin một trăm chào + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"vi_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + + types = verbalize | word + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/vi/verbalizers/whitelist.py new file mode 100644 index 000000000..954fc7028 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/whitelist.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, +) + + +class WhiteListFst(GraphFst): + """ + Finite state transducer for verbalizing whitelist for Vietnamese + e.g. tokens { name: "giờ" } -> giờ + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) + graph = ( + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", NEMO_SPACE), "", "", NEMO_SIGMA) + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/vi/verbalizers/word.py b/nemo_text_processing/text_normalization/vi/verbalizers/word.py new file mode 100644 index 000000000..4e11b3ed7 --- /dev/null +++ b/nemo_text_processing/text_normalization/vi/verbalizers/word.py @@ -0,0 +1,43 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.vi.graph_utils import ( + NEMO_NOT_QUOTE, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, + delete_space, +) + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing Vietnamese words. + e.g. tokens { name: "ngày" } -> ngày + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_NOT_QUOTE, 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", NEMO_SPACE), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv b/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv index 963b07d12..3848d54f9 100644 --- a/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv +++ b/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv @@ -70,3 +70,5 @@ … ‧ ﹏ +< +> \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv b/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv index 7f16f52a4..4eff86d48 100644 --- a/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv +++ b/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv @@ -5,3 +5,4 @@ × 乘 ÷ 除 ° 度 +- 减 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv b/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv index 6d45b4a3b..b1a8a832a 100644 --- a/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv +++ b/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv @@ -1,7 +1,5 @@ amu 原子质量 bar 巴 -° 度 -º 度 °c 摄氏度 °C 摄氏度 ºc 摄氏度 @@ -40,23 +38,6 @@ kw 千瓦 kW 千瓦 lb 磅 lbs 磅 -m2 平方米 -m² 平方米 -m3 立方米 -m³ 立方米 -mbps 兆比特每秒 -mg 毫克 -mhz 兆赫兹 -mi2 平方英里 -mi² 平方英里 -mi 英里 -min 分钟哦 -ml 毫升 -mm2 平方毫米 -mm² 平方毫米 -mol 摩尔 -mpa 兆帕 -mph 英里每小时 ng 纳克 nm 纳米 ns 纳秒 @@ -80,13 +61,7 @@ gb 吉字节 gpa 吉帕斯卡 gy 戈瑞 ha 公顷 -m 米 -mm 毫米 -ms 毫秒 -mv 毫伏 -mw 毫瓦 pg 皮克 ps 皮秒 s 秒 -ms 毫秒 g 克 diff --git a/nemo_text_processing/text_normalization/zh/data/measure/units_zh.tsv b/nemo_text_processing/text_normalization/zh/data/measure/units_zh.tsv deleted file mode 100644 index 5ca1dd9ab..000000000 --- a/nemo_text_processing/text_normalization/zh/data/measure/units_zh.tsv +++ /dev/null @@ -1,211 +0,0 @@ -匹 -张 -座 -回 -场 -尾 -条 -个 -首 -阙 -阵 -网 -炮 -顶 -丘 -棵 -只 -支 -袭 -辆 -挑 -担 -颗 -壳 -窠 -曲 -墙 -群 -腔 -砣 -座 -客 -贯 -扎 -捆 -刀 -令 -打 -手 -罗 -坡 -山 -岭 -江 -溪 -钟 -队 -单 -双 -对 -口 -头 -脚 -板 -跳 -枝 -件 -贴 -针 -线 -管 -名 -位 -身 -堂 -课 -本 -页 -家 -户 -层 -丝 -毫 -厘 -分 -钱 -两 -斤 -担 -铢 -石 -钧 -锱 -忽 -克 -毫 -厘 -分 -寸 -尺 -丈 -里 -寻 -常 -铺 -程 -米 -撮 -勺 -合 -升 -斗 -石 -盘 -碗 -碟 -叠 -桶 -笼 -盆 -盒 -杯 -钟 -斛 -锅 -簋 -篮 -盘 -桶 -罐 -瓶 -壶 -卮 -盏 -箩 -箱 -煲 -啖 -袋 -钵 -年 -月 -日 -季 -刻 -时 -周 -天 -秒 -分 -旬 -纪 -岁 -世 -更 -夜 -春 -夏 -秋 -冬 -代 -伏 -辈 -丸 -泡 -粒 -颗 -幢 -堆 -条 -根 -支 -道 -面 -片 -张 -颗 -块 -架 -千米 -分米 -厘米 -毫米 -微米 -纳米 -亿 -千万 -百万 -万 -千 -百 -亿块 -千万块 -百万块 -万块 -千块 -百块 -亿角 -千万角 -百万角 -万角 -千角 -百角 -亿毛 -千万毛 -百万毛 -万毛 -千毛 -百毛 -亿分 -千万分 -百万分 -万分 -千分 -百分 -亿元 -千万元 -百万元 -万元 -千元 -百元 diff --git a/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv index 88e6cc544..b80833507 100644 --- a/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv +++ b/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv @@ -168,7 +168,6 @@ Ft 匈牙利福林 ₪ 以色列谢克尔 J$ 牙买加元 лв 哈萨克斯坦腾格 -₩ 朝鲜园 лв 吉尔吉斯斯坦索姆 ₭ 老挝基普 ден 马其顿代纳尔 diff --git a/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv b/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv new file mode 100644 index 000000000..b949b9508 --- /dev/null +++ b/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv @@ -0,0 +1,9 @@ +1 一 +2 两 +3 三 +4 四 +5 五 +6 六 +7 七 +8 八 +9 九 diff --git a/nemo_text_processing/text_normalization/zh/data/number/suffix.tsv b/nemo_text_processing/text_normalization/zh/data/number/suffix.tsv new file mode 100644 index 000000000..f44c0e151 --- /dev/null +++ b/nemo_text_processing/text_normalization/zh/data/number/suffix.tsv @@ -0,0 +1,23 @@ +万 +十万 +百万 +千万 +亿 +十亿 +百亿 +千亿 +萬 +十萬 +百萬 +千萬 +億 +十億 +百億 +千億 +拾萬 +佰萬 +仟萬 +拾億 +佰億 +仟億 + diff --git a/nemo_text_processing/text_normalization/zh/data/number/teen.tsv b/nemo_text_processing/text_normalization/zh/data/number/teen.tsv new file mode 100644 index 000000000..52dc01917 --- /dev/null +++ b/nemo_text_processing/text_normalization/zh/data/number/teen.tsv @@ -0,0 +1,10 @@ +10 十 +11 十一 +12 十二 +13 十三 +14 十四 +15 十五 +16 十六 +17 十七 +18 十八 +19 十九 diff --git a/nemo_text_processing/text_normalization/zh/data/number/teen_alt.tsv b/nemo_text_processing/text_normalization/zh/data/number/teen_alt.tsv new file mode 100644 index 000000000..a48662621 --- /dev/null +++ b/nemo_text_processing/text_normalization/zh/data/number/teen_alt.tsv @@ -0,0 +1,10 @@ +10 一十 +11 一十一 +12 一十二 +13 一十三 +14 一十四 +15 一十五 +16 一十六 +17 一十七 +18 一十八 +19 一十九 diff --git a/nemo_text_processing/text_normalization/zh/data/number/ties.tsv b/nemo_text_processing/text_normalization/zh/data/number/ties.tsv new file mode 100644 index 000000000..2a73c0399 --- /dev/null +++ b/nemo_text_processing/text_normalization/zh/data/number/ties.tsv @@ -0,0 +1,8 @@ +2 二十 +3 三十 +4 四十 +5 五十 +6 六十 +7 七十 +8 八十 +9 九十 diff --git a/nemo_text_processing/text_normalization/zh/data/whitelist.tsv b/nemo_text_processing/text_normalization/zh/data/whitelist.tsv index 133143950..e8810f42a 100644 --- a/nemo_text_processing/text_normalization/zh/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/zh/data/whitelist.tsv @@ -77,12 +77,6 @@ C C t v CCTV kfc KFC K F C KFC Steam steam -phd 博士 -PhD 博士 -Dr. 医生 -Mr. 先生 -Mrs. 女士 -Ms. 小姐 O 2 O O to O O2O O to O P 2 P P to P @@ -161,4 +155,3 @@ cctv CCTV C C t v CCTV kfc KFC K F C KFC -Steam steam diff --git a/nemo_text_processing/text_normalization/zh/graph_utils.py b/nemo_text_processing/text_normalization/zh/graph_utils.py index c0c1a0792..69652b1f5 100644 --- a/nemo_text_processing/text_normalization/zh/graph_utils.py +++ b/nemo_text_processing/text_normalization/zh/graph_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# gaph_utils is kept due to the fatc that importing from en folders will cause import errors that the data file names have to be the same with what are in the en folder -import logging import os import string from pathlib import Path @@ -24,30 +22,40 @@ from pynini.export import export from pynini.lib import byte, pynutil, utf8 +from nemo_text_processing.utils.logging import logger + +# ghaph_utils is here since importing from en folders will cause import errors +# that the data file names have to be the same with what are in the en folder + NEMO_CHAR = utf8.VALID_UTF8_CHAR NEMO_DIGIT = byte.DIGIT NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() -NEMO_NON_BREAKING_SPACE = u"\u00A0" +NEMO_NON_BREAKING_SPACE = "\u00a0" NEMO_SPACE = " " -NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00A0").optimize() +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() - NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_NOT_ALPHA = pynini.difference(NEMO_SIGMA, NEMO_ALPHA).optimize() +NEMO_SPACE_CHAR = pynini.union(NEMO_CHAR, NEMO_SPACE) delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) insert_space = pynutil.insert(" ") delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) -def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): """ Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. @@ -59,7 +67,7 @@ def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): for rule, graph in graphs.items(): exporter[rule] = graph.optimize() exporter.close() - logging.info(f'Created {file_name}') + logger.info(f"Created {file_name}") class GraphFst: @@ -79,7 +87,7 @@ def __init__(self, name: str, kind: str, deterministic: bool = True): self._fst = None self.deterministic = deterministic - self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") if self.far_exist(): self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() @@ -90,14 +98,14 @@ def far_exist(self) -> bool: return self.far_path.exists() @property - def fst(self) -> 'pynini.FstLike': + def fst(self) -> "pynini.FstLike": return self._fst @fst.setter def fst(self, fst): self._fst = fst - def add_tokens(self, fst) -> 'pynini.FstLike': + def add_tokens(self, fst) -> "pynini.FstLike": """ Wraps class name around to given fst @@ -109,7 +117,7 @@ def add_tokens(self, fst) -> 'pynini.FstLike': """ return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") - def delete_tokens(self, fst) -> 'pynini.FstLike': + def delete_tokens(self, fst) -> "pynini.FstLike": """ Deletes class name wrap around output of given fst @@ -128,10 +136,10 @@ def delete_tokens(self, fst) -> 'pynini.FstLike': + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) -def convert_space(fst) -> 'pynini.FstLike': +def convert_space(fst) -> "pynini.FstLike": """ Converts space to nonbreaking space. Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" diff --git a/nemo_text_processing/text_normalization/zh/taggers/cardinal.py b/nemo_text_processing/text_normalization/zh/taggers/cardinal.py index 3d750e8b9..a0c3b587d 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/zh/taggers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,618 +14,179 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst -from nemo_text_processing.text_normalization.zh.utils import get_abs_path from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_DIGIT, GraphFst +from nemo_text_processing.text_normalization.zh.utils import get_abs_path + class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals, e.g. - '23' -> cardinal { integer: "二十三" } - -10000 -> cardinal { negative: "负" integer: "一万" } - +10000 -> cardinal { positive: "正" integer: "一万" } + Finite state transducer for classifying cardinals + e.g. 23 -> cardinal { integer: "二十三" } """ - def __init__(self, deterministic: bool = True, lm: bool = False): + def __init__(self, deterministic: bool = True): super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_digit_alt = pynini.string_file(get_abs_path("data/number/digit_alt.tsv")) + graph_ties = pynini.string_file(get_abs_path("data/number/ties.tsv")) + graph_teen = pynini.string_file(get_abs_path("data/number/teen.tsv")) + graph_teen_alt = pynini.string_file(get_abs_path("data/number/teen_alt.tsv")) + + alls = NEMO_DIGIT**2 | NEMO_DIGIT**1 + graph_all = ( + (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen_alt | graph_digit + ) # graph_all when within a larger number e.g., 316-> 三百一十六 instead of 三百十六 + + graph_all = alls @ graph_all + graph_all_alt = ( + (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen | graph_digit + ) # graph_all when at the head of the larger numbere.g., 13万 -> 十三万 instead of 一十三万 + graph_all_alt = alls @ graph_all_alt + + hundreds = NEMO_DIGIT**3 + graph_hundred_component = (graph_digit + pynutil.insert('百')) + pynini.union( + pynini.closure(pynutil.delete('0')), + (pynini.closure(pynutil.delete('0') + pynutil.insert('零')) + graph_all), + ) + graph_hundred = hundreds @ graph_hundred_component + + self.digit = graph_digit.optimize() + self.all = graph_all.optimize() + + thousands = NEMO_DIGIT**4 + graph_thousand_component = (graph_digit_alt + pynutil.insert('千')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_hundred_component, + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_thousand = thousands @ graph_thousand_component + + ten_thousands = NEMO_DIGIT**5 + graph_ten_thousand_component = (graph_digit_alt + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_ten_thousand = ten_thousands @ graph_ten_thousand_component + + hundred_thousands = NEMO_DIGIT**6 + hundred_thousands_position = NEMO_DIGIT**2 + hundred_thousands_position = hundred_thousands_position @ graph_all_alt + graph_hundred_thousand_component = (hundred_thousands_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component + + millions = NEMO_DIGIT**7 + million_position = NEMO_DIGIT**3 + million_position = million_position @ graph_hundred_component + graph_million_component = (million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_million = millions @ graph_million_component + + ten_millions = NEMO_DIGIT**8 + ten_million_position = NEMO_DIGIT**4 + ten_million_position = ten_million_position @ graph_thousand_component + graph_ten_million_component = (ten_million_position + pynutil.insert('万')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_thousand_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_ten_million = ten_millions @ graph_ten_million_component + + hundred_millions = NEMO_DIGIT**9 + graph_hundred_million_component = (graph_digit_alt + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + (pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_hundred_million = hundred_millions @ graph_hundred_million_component + + thousand_millions = NEMO_DIGIT**10 + thousand_millions_position = NEMO_DIGIT**2 + thousand_millions_position = thousand_millions_position @ graph_all_alt + graph_thousand_million_component = (thousand_millions_position + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + ((pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_thousand_million = thousand_millions @ graph_thousand_million_component + + ten_billions = NEMO_DIGIT**11 + ten_billions_position = NEMO_DIGIT**3 + ten_billions_position = ten_billions_position @ graph_hundred_component + graph_ten_billions_component = (ten_billions_position + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + ((pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_ten_billions = ten_billions @ graph_ten_billions_component + + hundred_billions = NEMO_DIGIT**12 + hundred_billions_position = NEMO_DIGIT**4 + hundred_billions_position = hundred_billions_position @ graph_thousand_component + graph_hundred_billions_component = (hundred_billions_position + pynutil.insert('亿')) + pynini.union( + pynini.closure(pynutil.delete('0')), + graph_ten_million_component, + (pynutil.delete('0') + pynutil.insert('零') + graph_million_component), + (pynutil.delete('00') + pynutil.insert('零') + graph_hundred_thousand_component), + (pynutil.delete('000') + pynutil.insert('零') + graph_ten_thousand_component), + (pynutil.delete('0000') + pynutil.insert('零') + graph_thousand_component), + ((pynutil.delete('00000') + pynutil.insert('零') + graph_hundred_component)), + (pynini.closure(pynutil.delete('0')) + pynutil.insert('零') + graph_all), + ) + graph_hundred_billions = hundred_billions @ graph_hundred_billions_component - # imports - zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) - digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) - digit_tens = pynini.string_file(get_abs_path("data/number/digit_tens.tsv")) - - # morphemes inserted + punctuation - tens_digit = pynutil.insert('十') - hundred_digit = pynutil.insert('百') - thousand_digit = pynutil.insert('千') - tenthousand_digit = pynutil.insert('万') - hundredmillion_digit = pynutil.insert('亿') - delete_punct = pynini.closure(pynutil.delete(',') | pynutil.delete('，')) - - # 十几; 10-19 - graph_teen = ( - pynini.closure(delete_punct) - + pynini.cross('1', '十') - + ( - (pynini.closure(delete_punct) + (pynini.closure(delete_punct) + digit)) - | (pynini.closure(delete_punct) + pynini.cross('0', '')) - ) - ) - - # 十几; 10-19 but when not alone, but within a larger number, (e.g, 119) - graph_teen_alt = ( - (pynini.closure(delete_punct) + (pynini.cross('1', '一十') + pynini.closure(delete_punct) + digit)) - | (pynini.closure(delete_punct) + pynini.cross('10', '一十')) - | (pynini.closure(delete_punct) + (pynini.cross('1,0', '一十') | pynini.cross('1，0', '一十'))) - ) # when the teen is not by itself but with in a larger number - - # 几十; 20-99 - graph_tens = ( - pynini.closure(delete_punct) - + (digit_tens + tens_digit + pynini.closure(delete_punct) + ((pynini.closure(delete_punct) + digit))) - ) | ( - digit_tens + tens_digit + (pynini.closure(delete_punct) + (pynini.cross('0', '') | pynini.cross(',0', ''))) - ) - - # 百; 100-999; hundreds - graph_hundred = ( - ( - digit - + ( - pynutil.delete('00') - | (pynutil.delete(',00') | pynutil.delete('，00')) - | (pynutil.delete('0,0') | pynutil.delete('0，0')) - ) - + hundred_digit - ) - | (digit + hundred_digit + (graph_tens | graph_teen_alt)) - | ( - digit - + hundred_digit - + ( - (pynini.cross(',0', '零') | pynini.cross('，0', '零')) - | pynini.cross('0', '零') - | (pynini.cross('0,', '零') | pynini.cross('0，', '零')) - ) - + digit - ) - ) - - # 千; 1000-9999; thousands - graph_thousand = ( - ( - digit - + ( - (pynutil.delete(',000') | pynutil.delete('000') | pynutil.delete('0,00') | pynutil.delete('00,0')) - | ( - pynutil.delete('，000') - | pynutil.delete('000') - | pynutil.delete('0，00') - | pynutil.delete('00，0') - ) - ) - + thousand_digit - ) - | (digit + pynini.closure(delete_punct) + thousand_digit + graph_hundred) - | ( - digit - + thousand_digit - + (pynini.cross('0', '零') | ((pynini.cross(',0', '零') | pynini.cross('，0', '零')))) - + (graph_tens | graph_teen_alt) - ) - | ( - digit - + pynini.closure(delete_punct) - + thousand_digit - + ( - pynini.cross('00', '零') - | (pynini.cross(',00', '零') | pynini.cross('，00', '零')) - | (pynini.cross('0,0', '零') | pynini.cross('0，0', '零')) - | (pynini.cross('00,', '零') | pynini.cross('00，', '零')) - ) - + digit - ) - ) - - # 万; 10000-99999; ten thousands - graph_tenthousand = ( - ( - digit - + (pynutil.delete('0000') | (pynutil.delete('0,000') | pynutil.delete('0，000'))) - + tenthousand_digit - ) - | (digit + tenthousand_digit + graph_thousand) - | ( - digit - + tenthousand_digit - + (pynini.cross('0', '零') | (pynini.cross('0,', '零') | pynini.cross('0，', '零'))) - + graph_hundred - ) - | ( - digit - + tenthousand_digit - + (pynini.cross('00', '零') | (pynini.cross('0,0', '零') | pynini.cross('0，0', '零'))) - + (graph_tens | graph_teen_alt) - ) - | ( - digit - + tenthousand_digit - + (pynini.cross('000', '零') | (pynini.cross('0,00', '零') | pynini.cross('0，00', '零'))) - + digit - ) - ) - - # 十万; 100000-999999; hundred thousands - graph_hundredthousand = ( - pynutil.add_weight( - ( - (graph_tens | graph_teen) - + tenthousand_digit - + (pynutil.delete('0000') | (pynutil.delete('0,000') | pynutil.delete('0，000'))) - ), - -0.1, - ) - | ((graph_tens | graph_teen) + tenthousand_digit + graph_thousand) - | ( - (graph_tens | graph_teen) - + tenthousand_digit - + (pynini.cross('0', '零') | (pynini.cross('0,', '零') | pynini.cross('0，', '零'))) - + graph_hundred - ) - | ( - (graph_tens | graph_teen) - + tenthousand_digit - + (pynini.cross('00', '零') | (pynini.cross('0,0', '零') | pynini.cross('0，0', '零'))) - + (graph_tens | graph_teen_alt) - ) - | ( - (graph_tens | graph_teen) - + tenthousand_digit - + (pynini.cross('000', '零') | (pynini.cross('0,00', '零') | pynini.cross('0，00', '零'))) - + digit - ) - ) - - # 百万; 1000000-9999999; millions - graph_million = ( - pynutil.add_weight( - ( - graph_hundred - + tenthousand_digit - + (pynutil.delete('0000') | (pynutil.delete('0,000') | pynutil.delete('0，000'))) - ), - -1.0, - ) - | (graph_hundred + tenthousand_digit + graph_thousand) - | ( - graph_hundred - + tenthousand_digit - + (pynini.cross('0', '零') | (pynini.cross('0,', '零') | pynini.cross('0，', '零'))) - + graph_hundred - ) - | ( - graph_hundred - + tenthousand_digit - + (pynini.cross('00', '零') | (pynini.cross('0,0', '零') | pynini.cross('0，0', '零'))) - + (graph_tens | graph_teen_alt) - ) - | ( - graph_hundred - + tenthousand_digit - + (pynini.cross('000', '零') | (pynini.cross('0,00', '零') | pynini.cross('0，00', '零'))) - + digit - ) - ) - - # 千万; 10000000-99999999; ten millions - graph_tenmillion = ( - pynutil.add_weight( - ( - graph_thousand - + (pynutil.delete('0000') | (pynutil.delete('0,000') | pynutil.delete('0，000'))) - + tenthousand_digit - ), - -1.0, - ) - | (graph_thousand + tenthousand_digit + graph_thousand) - | ( - graph_thousand - + tenthousand_digit - + (pynini.cross('0', '零') | (pynini.cross('0,', '零') | pynini.cross('0，', '零'))) - + graph_hundred - ) - | ( - graph_thousand - + tenthousand_digit - + (pynini.cross('00', '零') | (pynini.cross('0,0', '零') | pynini.cross('0，0', '零'))) - + (graph_tens | graph_teen_alt) - ) - | ( - graph_thousand - + tenthousand_digit - + (pynini.cross('000', '零') | (pynini.cross('0,00', '零') | pynini.cross('0，00', '零'))) - + digit - ) - ) - - # 亿; 100000000-999999999; hundred millions - graph_hundredmillion = ( - pynutil.add_weight( - ( - digit - + (pynutil.delete('00000000') | (pynutil.delete('00,000,000') | pynutil.delete('00，000，000'))) - + hundredmillion_digit - ), - -2.0, - ) - | pynutil.add_weight((digit + hundredmillion_digit + graph_tenmillion), -1.9) - | pynutil.add_weight((digit + hundredmillion_digit + pynutil.delete('0') + graph_million), -1.8) - | pynutil.add_weight( - (digit + hundredmillion_digit + pynutil.delete('00') + pynutil.insert('零') + graph_hundredthousand), - -1.7, - ) - | pynutil.add_weight( - ( - digit - + hundredmillion_digit - + (pynutil.delete('000') | (pynutil.delete('00,0') | pynutil.delete('00，0'))) - + pynutil.insert('零') - + graph_tenthousand - ), - -1.6, - ) - | pynutil.add_weight( - ( - digit - + hundredmillion_digit - + (pynutil.delete('0000') | (pynutil.delete('00,00') | pynutil.delete('00，00'))) - + pynutil.insert('零') - + graph_thousand - ), - -1.5, - ) - | pynutil.add_weight( - ( - digit - + hundredmillion_digit - + (pynutil.delete('00000') | (pynutil.delete('00,000,') | pynutil.delete('00，000，'))) - + pynutil.insert('零') - + graph_hundred - ), - -1.4, - ) - | pynutil.add_weight( - ( - digit - + hundredmillion_digit - + (pynutil.delete('000000') | (pynutil.delete('00,000,0') | pynutil.delete('00，000，0'))) - + pynutil.insert('零') - + (graph_tens | graph_teen_alt) - ), - -1.3, - ) - | pynutil.add_weight( - ( - digit - + hundredmillion_digit - + (pynutil.delete('0000000') | (pynutil.delete('00,000,00') | pynutil.delete('00，000，00'))) - + pynutil.insert('零') - + digit - ), - -1.2, - ) - ) - - # 十亿; 1000000000-9999999999; billions - graph_billion = ( - pynutil.add_weight( - ( - (graph_tens | graph_teen) - + (pynutil.delete('00000000') | (pynutil.delete('00,000,000') | pynutil.delete('00，000，000'))) - + hundredmillion_digit - ), - -2.0, - ) - | pynutil.add_weight(((graph_tens | graph_teen) + hundredmillion_digit + graph_tenmillion), -1.9) - | pynutil.add_weight( - ((graph_tens | graph_teen) + hundredmillion_digit + pynutil.delete('0') + graph_million), -1.8 - ) - | pynutil.add_weight( - ( - (graph_tens | graph_teen) - + hundredmillion_digit - + pynutil.delete('00') - + pynutil.insert('零') - + graph_hundredthousand - ), - -1.7, - ) - | pynutil.add_weight( - ( - (graph_tens | graph_teen) - + hundredmillion_digit - + (pynutil.delete('000') | (pynutil.delete('00,0') | pynutil.delete('00，0'))) - + pynutil.insert('零') - + graph_tenthousand - ), - -1.6, - ) - | pynutil.add_weight( - ( - (graph_tens | graph_teen) - + hundredmillion_digit - + (pynutil.delete('0000') | (pynutil.delete('00,00') | pynutil.delete('00，00'))) - + pynutil.insert('零') - + graph_thousand - ), - -1.5, - ) - | pynutil.add_weight( - ( - (graph_tens | graph_teen) - + hundredmillion_digit - + (pynutil.delete('00000') | (pynutil.delete('00,000,') | pynutil.delete('00，000，'))) - + pynutil.insert('零') - + graph_hundred - ), - -1.4, - ) - | pynutil.add_weight( - ( - (graph_tens | graph_teen) - + hundredmillion_digit - + (pynutil.delete('000000') | (pynutil.delete('00,000,0') | pynutil.delete('00，000，0'))) - + pynutil.insert('零') - + (graph_tens | graph_teen_alt) - ), - -1.3, - ) - | pynutil.add_weight( - ( - (graph_tens | graph_teen) - + hundredmillion_digit - + (pynutil.delete('0000000') | (pynutil.delete('00,000,00') | pynutil.delete('00，000，00'))) - + pynutil.insert('零') - + digit - ), - -1.2, - ) - ) - - # 百亿; 10000000000-99999999999; ten billions - graph_tenbillion = ( - pynutil.add_weight( - ( - graph_hundred - + (pynutil.delete('00000000') | (pynutil.delete('00,000,000') | pynutil.delete('00，000，000'))) - + hundredmillion_digit - ), - -2.0, - ) - | pynutil.add_weight((graph_hundred + hundredmillion_digit + graph_tenmillion), -1.9) - | pynutil.add_weight((graph_hundred + hundredmillion_digit + pynutil.delete('0') + graph_million), -1.8) - | pynutil.add_weight( - ( - graph_hundred - + hundredmillion_digit - + pynutil.delete('00') - + pynutil.insert('零') - + graph_hundredthousand - ), - -1.7, - ) - | pynutil.add_weight( - ( - graph_hundred - + hundredmillion_digit - + (pynutil.delete('000') | (pynutil.delete('00,0') | pynutil.delete('00，0'))) - + pynutil.insert('零') - + graph_tenthousand - ), - -1.6, - ) - | pynutil.add_weight( - ( - graph_hundred - + hundredmillion_digit - + (pynutil.delete('0000') | (pynutil.delete('00,00') | pynutil.delete('00，00'))) - + pynutil.insert('零') - + graph_thousand - ), - -1.5, - ) - | pynutil.add_weight( - ( - graph_hundred - + hundredmillion_digit - + (pynutil.delete('00000') | (pynutil.delete('00,000,') | pynutil.delete('00，000，'))) - + pynutil.insert('零') - + graph_hundred - ), - -1.4, - ) - | pynutil.add_weight( - ( - graph_hundred - + hundredmillion_digit - + (pynutil.delete('000000') | (pynutil.delete('00,000,0') | pynutil.delete('00，000，0'))) - + pynutil.insert('零') - + (graph_tens | graph_teen_alt) - ), - -1.3, - ) - | pynutil.add_weight( - ( - graph_hundred - + hundredmillion_digit - + (pynutil.delete('0000000') | (pynutil.delete('00,000,00') | pynutil.delete('00，000，00'))) - + pynutil.insert('零') - + digit - ), - -1.2, - ) - ) - - # 千亿; 100000000000-999999999999; hundred billions - graph_hundredbillion = ( - pynutil.add_weight( - ( - graph_thousand - + hundredmillion_digit - + (pynutil.delete('00000000') | (pynutil.delete('00,000,000') | pynutil.delete('00，000，000'))) - ), - -2.0, - ) - | pynutil.add_weight((graph_thousand + hundredmillion_digit + graph_tenmillion), -1.9) - | pynutil.add_weight((graph_thousand + hundredmillion_digit + pynutil.delete('0') + graph_million), -1.8) - | pynutil.add_weight( - ( - graph_thousand - + hundredmillion_digit - + pynutil.delete('00') - + pynutil.insert('零') - + graph_hundredthousand - ), - -1.7, - ) - | pynutil.add_weight( - ( - graph_thousand - + hundredmillion_digit - + (pynutil.delete('000') | (pynutil.delete('00,0') | pynutil.delete('00，0'))) - + pynutil.insert('零') - + graph_tenthousand - ), - -1.6, - ) - | pynutil.add_weight( - ( - graph_thousand - + hundredmillion_digit - + (pynutil.delete('0000') | (pynutil.delete('00,00') | pynutil.delete('00，00'))) - + pynutil.insert('零') - + graph_thousand - ), - -1.5, - ) - | pynutil.add_weight( - ( - graph_thousand - + hundredmillion_digit - + (pynutil.delete('00000') | (pynutil.delete('00,000,') | pynutil.delete('00，000，'))) - + pynutil.insert('零') - + graph_hundred - ), - -1.4, - ) - | pynutil.add_weight( - ( - graph_thousand - + hundredmillion_digit - + (pynutil.delete('000000') | (pynutil.delete('00,000,0') | pynutil.delete('00，000，0'))) - + pynutil.insert('零') - + (graph_tens | graph_teen_alt) - ), - -1.3, - ) - | pynutil.add_weight( - ( - graph_thousand - + hundredmillion_digit - + (pynutil.delete('0000000') | (pynutil.delete('00,000,00') | pynutil.delete('00，000，00'))) - + pynutil.insert('零') - + digit - ), - -1.2, - ) - ) - - suffix = pynini.union( - "万", - "十万", - "百万", - "千万", - "亿", - "十亿", - "百亿", - "千亿", - "萬", - "十萬", - "百萬", - "千萬", - "億", - "十億", - "百億", - "千億", - "拾萬", - "佰萬", - "仟萬", - "拾億", - "佰億", - "仟億", - "拾万", - "佰万", - "仟万", - "仟亿", - "佰亿", - "仟亿", - "万亿", - "萬億", - ) - graph_mandarin = pynini.closure( - ( - ( - digit - | graph_teen - | graph_tens - | graph_hundred - | graph_thousand - | graph_tenthousand - | graph_hundredthousand - ) - + suffix - ) - ) - - # combining all the graph above graph = pynini.union( - pynutil.add_weight(graph_hundredbillion, -2.0), - pynutil.add_weight(graph_tenbillion, -1.9), - pynutil.add_weight(graph_billion, -1.8), - pynutil.add_weight(graph_hundredmillion, -1.7), - pynutil.add_weight(graph_tenmillion, -1.6), - pynutil.add_weight(graph_million, -1.5), - pynutil.add_weight(graph_hundredthousand, -1.4), - pynutil.add_weight(graph_tenthousand, -1.3), - pynutil.add_weight(graph_thousand, -1.2), - pynutil.add_weight(graph_hundred, -1.1), - pynutil.add_weight(graph_tens, -1.0), - graph_teen, - digit, - zero, - ) - - # adding optional +(正)/-(负) signs - graph_sign = ( - (pynutil.insert("positive: \"") + pynini.accep("正") + pynutil.insert("\"")) - | (pynutil.insert("negative: \"") + pynini.accep("负") + pynutil.insert("\"")) - | (pynutil.insert("negative: \"") + pynini.cross("負", "负") + pynutil.insert("\"")) - | (pynutil.insert("negative: \"") + pynini.cross("-", "负") + pynutil.insert("\"")) - | (pynutil.insert("positive: \"") + pynini.cross("+", "正") + pynutil.insert("\"")) - ) - - graph_mandarin_sign = graph_sign + pynutil.insert(" ") + graph_mandarin - # final graph - final_graph_sign = ( - graph_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") - ) - final_graph_numbers_only = pynutil.insert("integer: \"") + graph + pynutil.insert("\"") - # imprted when building other grammars - self.just_cardinals = graph | graph_mandarin | final_graph_sign | graph_mandarin_sign - graph_mandarins = pynutil.insert("integer: \"") + graph_mandarin + pynutil.insert("\"") - - final_graph = final_graph_numbers_only | final_graph_sign | graph_mandarins | graph_mandarin_sign + graph_hundred_billions, + graph_ten_billions, + graph_thousand_million, + graph_hundred_million, + graph_ten_million, + graph_million, + graph_hundred_thousand, + graph_ten_thousand, + graph_thousand, + graph_hundred, + graph_all_alt, + graph_zero, + ) + self.just_cardinals = graph.optimize() + optional_sign = ( + pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("负", "-")) + pynutil.insert("\"") + ) + final_graph = ( + optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") + ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + + self.with_sign = final_graph.optimize() final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/taggers/date.py b/nemo_text_processing/text_normalization/zh/taggers/date.py index 849ada5a6..f5ea122e7 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/date.py +++ b/nemo_text_processing/text_normalization/zh/taggers/date.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class DateFst(GraphFst): @@ -31,7 +32,7 @@ class DateFst(GraphFst): 2002/02 -> is an error format according to the national standard 02/11 -> is an error format according to the national standard According to national standard, only when the year, month, and day are all exist, it is allowed to use symbols to separate them - + """ def __init__(self, deterministic: bool = True, lm: bool = False): @@ -57,12 +58,14 @@ def __init__(self, deterministic: bool = True, lm: bool = False): ) only_month = pynutil.insert("month: \"") + month + pynutil.delete('月') + pynutil.insert("\"") only_day = pynutil.insert("day: \"") + day + delete_day + pynutil.insert("\"") + # gh_1 graph_only_date = only_year | only_month | only_day year_month = only_year + pynutil.insert(' ') + only_month month_day = only_month + pynutil.insert(' ') + only_day - graph_all = only_year + pynutil.insert(' ') + only_month + pynutil.insert(' ') + only_day - graph_combination = year_month | month_day | graph_all + graph_ymd = only_year + pynutil.insert(' ') + only_month + pynutil.insert(' ') + only_day + # gh_2 + graph_combination = year_month | month_day | graph_ymd year_component = ( pynutil.insert("year: \"") @@ -72,8 +75,9 @@ def __init__(self, deterministic: bool = True, lm: bool = False): ) month_component = pynutil.insert("month: \"") + month + delete_sign + pynutil.insert("\"") day_component = pynutil.insert("day: \"") + day + pynutil.insert("\"") + # gp_3 graph_sign = year_component + pynutil.insert(' ') + month_component + pynutil.insert(' ') + day_component - + # gp_1+2+3 graph_all = graph_only_date | graph_sign | graph_combination prefix = ( @@ -85,11 +89,13 @@ def __init__(self, deterministic: bool = True, lm: bool = False): | pynini.accep('纪元前') ) prefix_component = pynutil.insert("era: \"") + prefix + pynutil.insert("\"") - graph_prefix = prefix_component + pynutil.insert(' ') + (pynutil.add_weight(graph_all, -2.0)) + # gp_prefix+(1,2,3) + graph_prefix = prefix_component + pynutil.insert(' ') + (graph_ymd | year_month | only_year) suffix_component = pynutil.insert("era: \"") + suffix + pynutil.insert("\"") - graph_suffix = (pynutil.add_weight(graph_all, -2.0)) + pynutil.insert(' ') + suffix_component - + # gp_suffix +(1,2,3) + graph_suffix = (graph_ymd | year_month | only_year) + pynutil.insert(' ') + suffix_component + # gp_4 graph_affix = graph_prefix | graph_suffix graph_suffix_year = ( diff --git a/nemo_text_processing/text_normalization/zh/taggers/decimal.py b/nemo_text_processing/text_normalization/zh/taggers/decimal.py index 3e9b118bd..713fd4ab2 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/zh/taggers/decimal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil def get_quantity(decimal): @@ -63,7 +64,7 @@ class DecimalFst(GraphFst): 0.5 -> decimal { integer_part: "零" fractional_part: "五" } 0.5万 -> decimal { integer_part: "零" fractional_part: "五" quantity: "万" } -0.5万 -> decimal { negative: "负" integer_part: "零" fractional_part: "五" quantity: "万"} - + Args: cardinal: CardinalFst """ @@ -72,19 +73,25 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = Fa super().__init__(name="decimal", kind="classify", deterministic=deterministic) cardinal_before_decimal = cardinal.just_cardinals - cardinal_after_decimal = pynini.string_file(get_abs_path("data/number/digit.tsv")) | pynini.closure( - pynini.cross('0', '零') - ) + cardinal_after_decimal = pynini.string_file(get_abs_path("data/number/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) + + graph_integer = pynutil.insert('integer_part: \"') + cardinal_before_decimal + pynutil.insert("\"") - decimal_point = pynini.closure(pynutil.delete('.'), 0, 1) - graph_integer = pynutil.insert("integer_part: \"") + cardinal_before_decimal + pynutil.insert("\"") graph_fraction = ( - pynutil.insert("fractional_part: \"") + pynini.closure(cardinal_after_decimal, 1) + pynutil.insert("\"") + pynutil.insert("fractional_part: \"") + + pynini.closure((pynini.closure(cardinal_after_decimal, 1) | (pynini.closure(zero, 1))), 1) + + pynutil.insert("\"") ) - graph_decimal = graph_integer + decimal_point + pynutil.insert(" ") + graph_fraction + graph_decimal = graph_integer + pynutil.delete('.') + pynutil.insert(" ") + graph_fraction + self.regular_decimal = graph_decimal.optimize() graph_sign = ( - (pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"负\"")) + pynutil.insert(" ")) + ( + pynini.closure(pynutil.insert("negative: \"") + pynini.cross("-", "负")) + + pynutil.insert("\"") + + pynutil.insert(" ") + ) ) | ( ( pynutil.insert('negative: ') @@ -97,14 +104,12 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = Fa graph_with_sign = graph_sign + graph_decimal graph_regular = graph_with_sign | graph_decimal - # graph_decimal_quantity = get_quantity(graph_decimal, cardinal.just_cardinals) graph_decimal_quantity = get_quantity(graph_decimal) graph_sign_quantity = graph_sign + graph_decimal_quantity graph_quantity = graph_decimal_quantity | graph_sign_quantity - # final_graph = graph_decimal | graph_sign | graph_decimal_quantity | graph_sign_quantity final_graph = graph_regular | graph_quantity - self.decimal = final_graph + self.decimal = final_graph.optimize() final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/taggers/fraction.py b/nemo_text_processing/text_normalization/zh/taggers/fraction.py index b0a302a8b..e3ad5b513 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/zh/taggers/fraction.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,11 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst +from nemo_text_processing.text_normalization.zh.utils import get_abs_path + class FractionFst(GraphFst): """ @@ -28,16 +30,17 @@ class FractionFst(GraphFst): 100分之1 -> tokens { fraction { denominator: "一百" numerator: "一"} } 百分之1 -> tokens { fraction { denominator: "百" numerator: "一"} } 98% -> tokens { fraction { denominator: "百" numerator: "九十八"} } - + Args: - cardinal: CardinalFst， decimal: DecimalFst + cardinal: CardinalFst, decimal: DecimalFst """ - def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True, lm: bool = False): + def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = False): super().__init__(name="fraction", kind="classify", deterministic=deterministic) graph_cardinals = cardinal.just_cardinals - graph_decimal = decimal.decimal + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) slash = pynutil.delete('/') morpheme = pynutil.delete('分之') @@ -74,7 +77,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = "仟亿", ) - integer_component = pynutil.insert("integer_part: \"") + graph_cardinals + pynutil.insert("\"") + integer_component = pynutil.insert('integer_part: \"') + graph_cardinals + pynutil.insert("\"") denominator_component = pynutil.insert("denominator: \"") + graph_cardinals + pynutil.insert("\"") numerator_component = pynutil.insert("numerator: \"") + graph_cardinals + pynutil.insert("\"") @@ -85,7 +88,8 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = + slash + pynutil.insert(' ') + denominator_component - ) + ) # 5又1/3 + graph_only_slash = numerator_component + slash + pynutil.insert(' ') + denominator_component graph_morpheme = (denominator_component + morpheme + pynutil.insert(' ') + numerator_component) | ( @@ -96,65 +100,68 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = + morpheme + pynutil.insert(' ') + numerator_component - ) + ) # 5又3分之1 graph_with_suffix = ( pynini.closure(pynutil.insert("denominator: \"") + suffix + pynutil.insert("\""), 0, 1) + morpheme + pynutil.insert(' ') + numerator_component - ) + ) # 万分之1 percentage = pynutil.delete('%') - graph_percentage = ( - numerator_component - + percentage - + pynutil.insert(' ') - + pynutil.insert("denominator: \"百") + + graph_decimal = ( + pynutil.insert('integer_part: \"') + + pynini.closure( + graph_cardinals + + pynutil.delete('.') + + pynutil.insert('点') + + pynini.closure((graph_digit | graph_zero), 1) + ) + pynutil.insert("\"") ) + graph_decimal_percentage = pynini.closure( + graph_decimal + percentage + pynutil.insert(' denominator: \"百"'), 1 + ) # 5.6% - graph_hundred = pynutil.delete('100%') + pynutil.insert('numerator: \"百\" denominator: \"百"') + graph_integer_percentage = pynini.closure( + (numerator_component) + percentage + pynutil.insert(' denominator: \"百"'), 1 + ) # 5% - graph_optional_sign = ( - (pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"负\""))) - | (pynini.closure(pynutil.insert("positive: ") + pynini.cross("+", "\"正\""))) - | (pynutil.insert("positive: ") + pynutil.insert("\"") + pynini.accep('正') + pynutil.insert("\"")) - | ( - pynutil.insert('negative: ') - + pynutil.insert("\"") - + (pynini.accep('负') | pynini.cross('負', '负')) - + pynutil.insert("\"") - ) - ) + graph_hundred = pynutil.delete('100%') + pynutil.insert('numerator: \"百\" denominator: \"百"') + # 100% - graph_decimals = ( - graph_decimal - + pynutil.insert(" ") - + percentage - + pynutil.insert("denominator: \"百") + graph_optional_sign = (pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"负\""))) | ( + pynutil.insert('negative: ') + + pynutil.insert("\"") + + (pynini.accep('负') | pynini.cross('負', '负')) + pynutil.insert("\"") ) - graph = ( - graph_with_integer - | graph_only_slash - | graph_morpheme - | graph_with_suffix - | graph_percentage - | graph_decimals - | pynutil.add_weight(graph_hundred, -3.0) + graph = pynini.union( + graph_with_integer, + graph_only_slash, + graph_morpheme, + graph_with_suffix, + graph_decimal_percentage, + graph_integer_percentage, + graph_hundred, ) graph_with_sign = ( (graph_optional_sign + pynutil.insert(" ") + graph_with_integer) | (graph_optional_sign + pynutil.insert(" ") + graph_only_slash) | (graph_optional_sign + pynutil.insert(" ") + graph_morpheme) | (graph_optional_sign + pynutil.insert(" ") + graph_with_suffix) - | (graph_optional_sign + pynutil.insert(" ") + graph_percentage) - | pynutil.add_weight((graph_optional_sign + pynutil.insert(" ") + graph_hundred), -3.0) + | (graph_optional_sign + pynutil.insert(" ") + graph_integer_percentage) + | (graph_optional_sign + pynutil.insert(" ") + graph_decimal_percentage) + | (graph_optional_sign + pynutil.insert(" ") + graph_hundred) ) - final_graph = graph | graph_with_sign + final_graph = graph | pynutil.add_weight(graph_with_sign, -3.0) + + self.just_fractions = graph.optimize() + self.fractions = final_graph.optimize() final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/taggers/math_symbol.py b/nemo_text_processing/text_normalization/zh/taggers/math_symbol.py deleted file mode 100644 index 239466954..000000000 --- a/nemo_text_processing/text_normalization/zh/taggers/math_symbol.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst -from nemo_text_processing.text_normalization.zh.taggers.cardinal import CardinalFst -from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil - - -class MathSymbol(GraphFst): - ''' - + -> tokens { sign: "加" } - ''' - - def __init__(self, deterministic: bool = True, lm: bool = False): - super().__init__(name="sign", kind="classify", deterministic=deterministic) - ''' - add your sign in data/math/symbol.tsv,this graph just convert sigh to character,you can add more - cases with detailed cases - ''' - score_sign = pynini.string_file(get_abs_path("data/math/score.tsv")) | pynini.string_file( - get_abs_path("data/math/symbol.tsv") - ) - score = ( - pynutil.insert("score: \"") - + pynini.closure(score_sign, 0, 1) - + CardinalFst().just_cardinals - + score_sign - + CardinalFst().just_cardinals - + pynutil.insert("\"") - ) - graph = score - self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/taggers/measure.py b/nemo_text_processing/text_normalization/zh/taggers/measure.py index f4d3c8688..1ec47aae9 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/measure.py +++ b/nemo_text_processing/text_normalization/zh/taggers/measure.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,39 +11,66 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, insert_space from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil -class Measure(GraphFst): +class MeasureFst(GraphFst): ''' - 1kg -> tokens { measure { cardinal { integer: "一" } units: "千克" } } + 1kg -> tokens { measure { cardinal { integer: "一" } units: "千克" } } ''' - def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True, lm: bool = False): + def __init__( + self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True, lm: bool = False + ): super().__init__(name="measure", kind="classify", deterministic=deterministic) units_en = pynini.string_file(get_abs_path("data/measure/units_en.tsv")) - units_zh = pynini.string_file(get_abs_path("data/measure/units_zh.tsv")) - graph_cardinal = cardinal.just_cardinals - integer_component = pynutil.insert("integer: \"") + graph_cardinal + pynutil.insert("\"") - unit_component = pynutil.insert("units: \"") + (units_en | units_zh) + pynutil.insert("\"") - graph_cardinal_measure = integer_component + insert_space + unit_component + graph_cardinal = cardinal.with_sign + graph_decimal = decimal.decimal + + # these units ared added due to falures when running Sparrow Hawk tests that "ms" would be processed as "m" and "s" left outside of the tagegr + units = ( + pynini.cross("ms", "毫秒") + | pynini.cross("m²", "平方米") + | pynini.cross("m2", "平方米") + | pynini.cross("m²", "平方米") + | pynini.cross("m³", "立方米") + | pynini.cross("mbps", "兆比特每秒") + | pynini.cross("mg", "毫克") + | pynini.cross("mhz", "兆赫兹") + | pynini.cross("mi2", "平方英里") + | pynini.cross("mi²", "平方英里") + | pynini.cross("mi", "英里") + | pynini.cross("min", "分钟") + | pynini.cross("ml", "毫升") + | pynini.cross("mm2", "平方毫米") + | pynini.cross("mm²", "平方毫米") + | pynini.cross("mol", "摩尔") + | pynini.cross("mpa", "兆帕") + | pynini.cross("mph", "英里每小时") + | pynini.cross("mm", "毫米") + | pynini.cross("mv", "毫伏") + | pynini.cross("mw", "毫瓦") + ) + + unit_component = pynutil.insert("units: \"") + (units_en | units) + pynutil.insert("\"") - decimal = decimal.decimal - graph_decimal = ( - decimal + insert_space + pynutil.insert("units: \"") + (units_en | units_zh) + pynutil.insert("\"") + graph_cardinal_measure = pynini.closure( + (pynutil.insert("cardinal { ") + graph_cardinal + pynutil.insert(" } ") + insert_space + unit_component), 1 ) - graph_sign = ( - (pynutil.insert("negative: \"") + pynini.accep("负") + pynutil.insert("\"")) - | (pynutil.insert("negative: \"") + pynini.cross("負", "负") + pynutil.insert("\"")) - | (pynutil.insert("negative: \"") + pynini.cross("-", "负") + pynutil.insert("\"")) + graph_decimal_measure = pynini.closure( + (pynutil.insert("decimal { ") + graph_decimal + pynutil.insert(" } ") + unit_component), 1 ) - graph = pynini.closure(graph_sign + insert_space) + (graph_cardinal_measure | graph_decimal) + graph_measures = graph_decimal_measure | graph_cardinal_measure - self.fst = self.add_tokens(graph).optimize() + final_graph = self.add_tokens(graph_measures) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/taggers/money.py b/nemo_text_processing/text_normalization/zh/taggers/money.py index b637bb2a8..d0c6d2ca5 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/money.py +++ b/nemo_text_processing/text_normalization/zh/taggers/money.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,43 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil + +suffix = pynini.union( + "万", + "十万", + "百万", + "千万", + "亿", + "十亿", + "百亿", + "千亿", + "萬", + "十萬", + "百萬", + "千萬", + "億", + "十億", + "百億", + "千億", + "拾萬", + "佰萬", + "仟萬", + "拾億", + "佰億", + "仟億", + "拾万", + "佰万", + "仟万", + "仟亿", + "佰亿", + "仟亿", + "万亿", + "萬億", +) class MoneyFst(GraphFst): @@ -26,18 +60,19 @@ class MoneyFst(GraphFst): '23美元' -> money { integer: "二十三" currency: "美元" } """ - def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True, lm: bool = False): + def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = False): super().__init__(name="money", kind="classify", deterministic=deterministic) cardinal = cardinal.just_cardinals - decimal = decimal.decimal currency = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) currency_mandarin = pynini.string_file(get_abs_path("data/money/currency_mandarin.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv")) # regular money gramamr with currency symbols $1000 currency_component = pynutil.insert("currency: \"") + currency + pynutil.insert("\"") - number_component = pynutil.insert("integer: \"") + cardinal + pynutil.insert("\"") + number_component = pynutil.insert("integer_part: \"") + (cardinal | (cardinal + suffix)) + pynutil.insert("\"") graph_regular_money = currency_component + pynutil.insert(" ") + number_component # 块 元 毛 with optional symbols @@ -53,8 +88,8 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = currency_mandarin_component = pynutil.insert("currency: \"") + currency_mandarin + pynutil.insert("\"") unit_components = ( (pynutil.insert("currency: \"") + unit_major + pynutil.insert("\"")) - | (pynutil.insert("currency_major: \"") + unit_minor + pynutil.insert("\"")) - | (pynutil.insert("currency_minor: \"") + unit_minor_alt + pynutil.insert("\"")) + | (pynutil.insert("currency_maj: \"") + unit_minor + pynutil.insert("\"")) + | (pynutil.insert("currency_min: \"") + unit_minor_alt + pynutil.insert("\"")) ) graph_unit_only = ( @@ -69,15 +104,36 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = graph_mandarin_money = number_component + pynutil.insert(" ") + currency_mandarin_component # larger money as decimals - graph_decimal_money = (decimal + pynutil.insert(" ") + currency_mandarin_component) | ( - currency_component + pynutil.insert(" ") + decimal + graph_decimal = ( + pynutil.insert('integer_part: \"') + + ( + pynini.closure(cardinal, 1) + + pynutil.delete('.') + + pynutil.insert('点') + + pynini.closure((graph_digit | graph_zero), 1) + ) + + pynutil.insert("\"") + ) + graph_decimal_money = ( + pynini.closure(graph_decimal, 1) + + pynini.closure((pynutil.insert(' quantity: \"') + suffix + pynutil.insert('\"')), 0, 1) + + pynutil.insert(" ") + + pynini.closure(currency_mandarin_component, 1) + ) | ( + pynini.closure(currency_component, 1) + + pynutil.insert(" ") + + pynini.closure(graph_decimal, 1) + + pynini.closure( + (pynutil.insert(" ") + pynutil.insert('quantity: \"') + suffix + pynutil.insert('\"')), 0, 1 + ) ) graph = ( - graph_regular_money | graph_units | pynutil.add_weight(graph_mandarin_money, -3.0) | graph_decimal_money + graph_regular_money + | graph_units + | pynutil.add_weight(graph_mandarin_money, -3.0) + | pynutil.add_weight(graph_decimal_money, -1.0) ) - final_graph = graph - - final_graph = self.add_tokens(final_graph) + final_graph = self.add_tokens(graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/taggers/ordinal.py b/nemo_text_processing/text_normalization/zh/taggers/ordinal.py index 88774edca..e09dd8047 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/zh/taggers/ordinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst + class OrdinalFst(GraphFst): """ @@ -27,8 +28,8 @@ class OrdinalFst(GraphFst): cardinal: CardinalFst """ - def __init__(self, cardinal: GraphFst): - super().__init__(name="ordinal", kind="verbalize") + def __init__(self, cardinal: GraphFst, deterministic: bool = True, lm: bool = False): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) graph_cardinal = cardinal.just_cardinals morpheme = pynini.accep('第') diff --git a/nemo_text_processing/text_normalization/zh/taggers/preprocessor.py b/nemo_text_processing/text_normalization/zh/taggers/preprocessor.py index 57d672dc6..b283f3444 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/preprocessor.py +++ b/nemo_text_processing/text_normalization/zh/taggers/preprocessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,25 +11,30 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_SIGMA, GraphFst from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil -class PreProcessor(GraphFst): +class PreProcessorFst(GraphFst): ''' - Preprocessing of TN: - 1. interjections removal such as '啊, 呃' - 2. fullwidth -> halfwidth char conversion - 好啊 -> 好 - 呃对 -> 对 - ： -> : - ； -> ; + Preprocessing of TN: + 1. interjections removal such as '啊, 呃' + 2. fullwidth -> halfwidth char conversion + 好啊 -> 好 + 呃对 -> 对 + ： -> : + ； -> ; ''' def __init__( - self, remove_interjections: bool = True, fullwidth_to_halfwidth: bool = True, + self, + remove_interjections: bool = True, + fullwidth_to_halfwidth: bool = True, ): super().__init__(name="PreProcessor", kind="processor") diff --git a/nemo_text_processing/text_normalization/zh/taggers/punctuation.py b/nemo_text_processing/text_normalization/zh/taggers/punctuation.py index 0d24a09fb..d6920c75d 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/punctuation.py +++ b/nemo_text_processing/text_normalization/zh/taggers/punctuation.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. + import sys from unicodedata import category import pynini -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst -from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels from pynini.examples import plurals from pynini.lib import pynutil +from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels + class PunctuationFst(GraphFst): """ @@ -35,7 +37,7 @@ class PunctuationFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="punctuation", kind="classify", deterministic=deterministic) - s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"。，；-《》“”" + s = "!#%&\'()*+,-/:;<=>?@^_`{|}~\"。，；-《》“”" punct_symbols_to_exclude = ["[", "]"] punct_unicode = [ diff --git a/nemo_text_processing/text_normalization/zh/taggers/time.py b/nemo_text_processing/text_normalization/zh/taggers/time.py index 87a01bc9b..b0248d5c3 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/time.py +++ b/nemo_text_processing/text_normalization/zh/taggers/time.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -40,9 +41,9 @@ def __init__(self, deterministic: bool = True, lm: bool = False): # gramamr for time, separated by colons 05:03:13 symbol = pynutil.delete(":") | pynutil.delete("：") - hour_component = pynutil.insert("hour: \"") + hour + pynutil.insert('点') + pynutil.insert("\"") - minute_component = pynutil.insert("minute: \"") + minute + pynutil.insert('分') + pynutil.insert("\"") - second_component = pynutil.insert("second: \"") + second + pynutil.insert('秒') + pynutil.insert("\"") + hour_component = pynutil.insert("hours: \"") + hour + pynutil.insert('点') + pynutil.insert("\"") + minute_component = pynutil.insert("minutes: \"") + minute + pynutil.insert('分') + pynutil.insert("\"") + second_component = pynutil.insert("seconds: \"") + second + pynutil.insert('秒') + pynutil.insert("\"") # combining 3 components hour_minute_second = ( hour_component @@ -74,12 +75,12 @@ def __init__(self, deterministic: bool = True, lm: bool = False): minute_duration = pynini.accep("分钟") | pynini.accep('刻') | pynini.accep('刻钟') second_duration = pynini.accep("秒钟") | pynini.cross('秒鐘', '秒钟') | pynini.accep('秒') # combining two above - hour_component = pynutil.insert("hour: \"") + hour + (hour_clock | hour_duration) + pynutil.insert("\"") + hour_component = pynutil.insert("hours: \"") + hour + (hour_clock | hour_duration) + pynutil.insert("\"") minute_component = ( - pynutil.insert("minute: \"") + minute + (minute_clock | minute_duration) + pynutil.insert("\"") + pynutil.insert("minutes: \"") + minute + (minute_clock | minute_duration) + pynutil.insert("\"") ) second_component = ( - pynutil.insert("second: \"") + second + (second_clock | second_duration) + pynutil.insert("\"") + pynutil.insert("seconds: \"") + second + (second_clock | second_duration) + pynutil.insert("\"") ) hour_minute = hour_component + pynutil.insert(' ') + minute_component hour_second = hour_component + pynutil.insert(' ') + second_component @@ -96,7 +97,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): ) # gramamr for time, back count; 五点差n分n秒 - backcount = pynutil.insert("verb: \"") + pynini.accep('差') + pynutil.insert("\"") + backcount = pynutil.insert("morphosyntactic_features: \"") + pynini.accep('差') + pynutil.insert("\"") graph_hour = ( ( pynini.closure(backcount) diff --git a/nemo_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py index 3ed24aa8f..3a0b28aeb 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/zh/taggers/tokenize_and_classify.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,38 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging + import os import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import ( - NEMO_CHAR, - NEMO_DIGIT, - GraphFst, - delete_extra_space, - delete_space, - generator_main, -) +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, generator_main from nemo_text_processing.text_normalization.zh.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.zh.taggers.date import DateFst from nemo_text_processing.text_normalization.zh.taggers.decimal import DecimalFst from nemo_text_processing.text_normalization.zh.taggers.fraction import FractionFst -from nemo_text_processing.text_normalization.zh.taggers.math_symbol import MathSymbol -from nemo_text_processing.text_normalization.zh.taggers.measure import Measure +from nemo_text_processing.text_normalization.zh.taggers.measure import MeasureFst from nemo_text_processing.text_normalization.zh.taggers.money import MoneyFst from nemo_text_processing.text_normalization.zh.taggers.ordinal import OrdinalFst -from nemo_text_processing.text_normalization.zh.taggers.preprocessor import PreProcessor from nemo_text_processing.text_normalization.zh.taggers.punctuation import PunctuationFst from nemo_text_processing.text_normalization.zh.taggers.time import TimeFst from nemo_text_processing.text_normalization.zh.taggers.whitelist import WhiteListFst -from nemo_text_processing.text_normalization.zh.taggers.word import Char -from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.taggers.word import WordFst class ClassifyFst(GraphFst): """ - Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased. - For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. More details to deployment at NeMo/tools/text_processing_deployment. Args: @@ -58,7 +50,7 @@ class ClassifyFst(GraphFst): def __init__( self, input_case: str, - deterministic: bool = False, + deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False, whitelist: str = None, @@ -69,85 +61,40 @@ def __init__( if cache_dir is not None and cache_dir != "None": os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" - far_file = os.path.join( - # cache_dir, f"_{input_case}_zh_tn_{deterministic}_deterministic{whitelist_file}.far" - cache_dir, - f"_{input_case}_zh_tn_{deterministic}_deterministic_{whitelist_file}.far", - ) + far_file = os.path.join(cache_dir, f"zh_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] - no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) - self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() - logging.info(f"ClassifyFst.fst was restored from {far_file}.") else: - logging.info(f"Creating ClassifyFst grammars. This might take some time...") - - cardinal = CardinalFst() - cardinal_graph = cardinal.fst - - ordinal = OrdinalFst(cardinal=cardinal) - ordinal_graph = ordinal.fst - - decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) - decimal_graph = decimal.fst - - fraction = FractionFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic) - fraction_graph = fraction.fst - + cardinal = CardinalFst(deterministic=deterministic) date = DateFst(deterministic=deterministic) - date_graph = date.fst - - word_graph = Char(deterministic=deterministic).fst - - self.time = TimeFst(deterministic=deterministic) - time_graph = self.time.fst - - money = MoneyFst(cardinal=cardinal, decimal=decimal, deterministic=deterministic) - money_graph = money.fst - - self.math = MathSymbol(deterministic=deterministic) - math_graph = self.math.fst - - self.measure = Measure(cardinal=cardinal, decimal=decimal, deterministic=deterministic) - measure_graph = self.measure.fst - - self.whitelist = WhiteListFst(input_case=input_case, deterministic=deterministic, input_file=whitelist) - whitelist_graph = self.whitelist.fst - punct_graph = PunctuationFst(deterministic=deterministic).fst - - classify = ( - pynutil.add_weight(whitelist_graph, 1.001) - | pynutil.add_weight(cardinal_graph, -2.0) - | pynutil.add_weight(time_graph, 1.1) - | pynutil.add_weight(fraction_graph, -1.1) - | pynutil.add_weight(date_graph, -1.0) - | pynutil.add_weight(ordinal_graph, 1.1) - | pynutil.add_weight(decimal_graph, -1.0) - | pynutil.add_weight(money_graph, -1.1) - | pynutil.add_weight(math_graph, 1.1) - | pynutil.add_weight(measure_graph, -1.1) - | pynutil.add_weight(word_graph, 1.1) - ) - - classify |= pynutil.add_weight(word_graph, 100) - - punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") - token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") - token_plus_punct = ( - pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) + decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) + time = TimeFst(deterministic=deterministic) + fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + whitelist = WhiteListFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) + punctuation = PunctuationFst(deterministic=deterministic) + + classify = pynini.union( + pynutil.add_weight(date.fst, 1.1), + pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(money.fst, 1.1), + pynutil.add_weight(measure.fst, 1.05), + pynutil.add_weight(time.fst, 1.1), + pynutil.add_weight(whitelist.fst, 1.1), + pynutil.add_weight(cardinal.fst, 1.1), + pynutil.add_weight(decimal.fst, 3.05), + pynutil.add_weight(ordinal.fst, 1.1), + pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(word.fst, 100), ) - graph = token_plus_punct + pynini.closure(pynutil.add_weight(delete_extra_space, 1.1) + token_plus_punct) - graph = delete_space + graph + delete_space - - # self.fst = graph.optimize() - tagger = graph.optimize() - preprocessor = PreProcessor(remove_interjections=True, fullwidth_to_halfwidth=True,) - self.fst = preprocessor.fst @ tagger + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") + tagger = pynini.closure(token, 1) - no_digits = pynini.closure(pynini.difference(NEMO_CHAR, NEMO_DIGIT)) - self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize() + self.fst = tagger if far_file: generator_main(far_file, {"tokenize_and_classify": self.fst}) - logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/zh/taggers/whitelist.py b/nemo_text_processing/text_normalization/zh/taggers/whitelist.py index 3b3c745d6..9015bd047 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/zh/taggers/whitelist.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, convert_space from nemo_text_processing.text_normalization.zh.utils import get_abs_path, load_labels -from pynini.lib import pynutil class WhiteListFst(GraphFst): @@ -32,33 +33,25 @@ class WhiteListFst(GraphFst): input_file: path to a file with whitelist replacements """ - def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None): + def __init__(self, deterministic: bool = True, input_file: str = None): super().__init__(name="whitelist", kind="classify", deterministic=deterministic) - def _get_whitelist_graph(input_case, file): + def _get_whitelist_graph(file): whitelist = load_labels(file) - if input_case == "lower_cased": - whitelist = [[x[0].lower()] + x[1:] for x in whitelist] graph = pynini.string_map(whitelist) return graph - graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv")) - if not deterministic and input_case != "lower_cased": - graph |= pynutil.add_weight( - _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")), weight=0.0001 - ) + graph = _get_whitelist_graph(get_abs_path("data/whitelist.tsv")) + + graph |= pynutil.add_weight(_get_whitelist_graph(get_abs_path("data/whitelist.tsv")), weight=0.0001) if input_file: - whitelist_provided = _get_whitelist_graph(input_case, input_file) + whitelist_provided = _get_whitelist_graph(input_file) if not deterministic: graph |= whitelist_provided else: graph = whitelist_provided - if not deterministic: - units_graph = _get_whitelist_graph(input_case, file=get_abs_path("data/measure/measurements.tsv")) - graph |= units_graph - self.graph = graph self.final_graph = convert_space(self.graph).optimize() self.fst = (pynutil.insert("name: \"") + self.final_graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/zh/taggers/word.py b/nemo_text_processing/text_normalization/zh/taggers/word.py index 24851c5b1..4e3b42b00 100644 --- a/nemo_text_processing/text_normalization/zh/taggers/word.py +++ b/nemo_text_processing/text_normalization/zh/taggers/word.py @@ -1,29 +1,34 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_SPACE, GraphFst -from pynini.lib import pynutil - - -class Char(GraphFst): - ''' - 你 -> char { name: "你" } - ''' - - def __init__(self, deterministic: bool = True, lm: bool = False): - super().__init__(name="char", kind="classify", deterministic=deterministic) - - graph = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") - self.fst = graph.optimize() +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying word. + e.g. dormir -> tokens { name: "dormir" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + NEMO_NOT_QUOTE + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/zh/utils.py b/nemo_text_processing/text_normalization/zh/utils.py index 2d78a8ea0..175aba206 100644 --- a/nemo_text_processing/text_normalization/zh/utils.py +++ b/nemo_text_processing/text_normalization/zh/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ def get_abs_path(rel_path): Args: rel_path: relative path to this file - + Returns absolute path """ return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path @@ -43,14 +43,14 @@ def load_labels(abs_path): Returns dictionary of mappings """ - label_tsv = open(abs_path, encoding="utf-8") - labels = list(csv.reader(label_tsv, delimiter="\t")) + with open(abs_path, encoding="utf-8") as label_tsv: + labels = list(csv.reader(label_tsv, delimiter="\t")) return labels def augment_labels_with_punct_at_end(labels): """ - augments labels: if key ends on a punctuation that value does not have, add a new label + augments labels: if key ends on a punctuation that value does not have, add a new label where the value maintains the punctuation Args: diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/zh/verbalizers/cardinal.py index a63f0756b..1a28241af 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/cardinal.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class CardinalFst(GraphFst): """ @@ -29,7 +30,7 @@ class CardinalFst(GraphFst): def __init__(self, deterministic: bool = True, lm: bool = False): super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) - delete_sign = pynini.cross("negative: \"负\"", "负") | pynini.cross("positive: \"正\"", "正") + delete_sign = pynini.cross("negative: \"-\"", "负") delete_integer = ( pynutil.delete("integer: ") + pynutil.delete("\"") @@ -43,6 +44,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): ) graph_sign = delete_sign + delete_space + delete_integer final_graph = delete_integer | graph_sign | graph_mandarin + self.numbers = final_graph delete_tokens = self.delete_tokens(final_graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/date.py b/nemo_text_processing/text_normalization/zh/verbalizers/date.py index b968c4b92..f69f4a797 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/date.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/date.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. + import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DateFst(GraphFst): """ @@ -54,6 +56,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): optional_era = ( pynutil.delete("era: ") + pynutil.delete("\"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") ) + graph_date = ( pynini.closure(year_component) + pynini.closure(delete_space) @@ -61,7 +64,20 @@ def __init__(self, deterministic: bool = True, lm: bool = False): + pynini.closure(delete_space) + pynini.closure(day_component) ) - graph_date_era = optional_era + delete_space + graph_date + + graph_date_era = pynini.union( + (optional_era + delete_space + year_component), + (optional_era + delete_space + year_component + delete_space + month_component), + ( + optional_era + + delete_space + + year_component + + delete_space + + month_component + + delete_space + + day_component + ), + ) graph_date_all = graph_date | graph_date_era @@ -83,6 +99,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): ) final_graph = graph_date_all | graph_range + # final_graph = optional_era + delete_space + year_component delete_tokens = self.delete_tokens(final_graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/decimal.py b/nemo_text_processing/text_normalization/zh/verbalizers/decimal.py index 4f1f8980f..795ab01a6 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/decimal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class DecimalFst(GraphFst): """ @@ -29,21 +30,9 @@ class DecimalFst(GraphFst): def __init__(self, deterministic: bool = True, lm: bool = False): super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) - integer = ( - pynutil.delete("integer_part:") - + delete_space - + pynutil.delete("\"") - + pynini.closure(NEMO_NOT_QUOTE, 1) - + pynutil.delete("\"") - ) + integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") - fractional = ( - pynutil.delete("fractional_part:") - + delete_space - + pynutil.delete("\"") - + pynini.closure(NEMO_NOT_QUOTE, 1) - + pynutil.delete("\"") - ) + fractional = pynutil.delete("fractional_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") quantity = ( pynutil.delete("quantity:") @@ -62,6 +51,7 @@ def __init__(self, deterministic: bool = True, lm: bool = False): ) graph = integer + delete_space + pynutil.insert("点") + fractional + self.decimal_regular = graph graph_quantity = graph + delete_space + quantity graph_regular = graph | graph_quantity diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/fraction.py b/nemo_text_processing/text_normalization/zh/verbalizers/fraction.py index 96aff9492..c2a719c16 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/fraction.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class FractionFst(GraphFst): """ @@ -71,10 +72,19 @@ def __init__(self, decimal: GraphFst, deterministic: bool = True, lm: bool = Fal ) graph_no_integer = denominator_part + delete_space + pynutil.insert('分之') + numerator_part graph = graph_with_integer | graph_no_integer - graph_with_sign = sign_part + delete_space + graph - graph_with_decimal = denominator_part + delete_space + pynutil.insert('分之') + graph_decimal + + graph_with_decimal = ( + denominator_part + + delete_space + + pynutil.insert('分之') + + pynutil.delete("integer_part: \"") + + pynini.closure(NEMO_NOT_QUOTE) + + pynutil.delete("\"") + ) + graph_with_sign = sign_part + delete_space + (graph | graph_with_decimal) final_graph = graph_with_sign | graph | graph_with_decimal + self.fraction = final_graph delete_tokens = self.delete_tokens(final_graph) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/measure.py b/nemo_text_processing/text_normalization/zh/verbalizers/measure.py index 19c190248..4487c6449 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/measure.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,40 +11,61 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space -class Measure(GraphFst): + +class MeasureFst(GraphFst): ''' - tokens { measure { cardinal: "一" } units: "千克" } } -> 一千克 + tokens { measure { cardinal: "一" } units: "千克" } } -> 一千克 ''' - def __init__(self, deterministic: bool = True, lm: bool = False): + def __init__( + self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True, lm: bool = False + ): super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + cardinal = cardinal.numbers + decimal = decimal.decimal_component sign_component = pynutil.delete("negative: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") - integer_component = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") unit_component = pynutil.delete("units: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") - cardinal_graph = integer_component + delete_space + unit_component + graph_cardinal = ( + pynutil.delete("cardinal { ") + cardinal + pynutil.delete(" } ") + delete_space + unit_component + ) - decimal_graph = ( - pynutil.delete("integer_part: \"") - + pynini.closure(NEMO_NOT_QUOTE) - + pynutil.delete("\"") - + pynutil.insert("点") + graph_decimal = ( + pynutil.delete("decimal {") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + decimal + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + pynutil.delete("}") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + delete_space - + pynutil.delete("fractional_part: \"") - + pynini.closure(NEMO_NOT_QUOTE, 0) - + pynutil.delete("\"") + + unit_component + ) + + graph_fraction = ( + pynutil.delete("fraction {") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + fraction.fraction + + pynini.closure(pynutil.delete(NEMO_SPACE)) + + pynutil.delete("}") + + pynini.closure(pynutil.delete(NEMO_SPACE)) + delete_space - + pynutil.delete("units: \"") - + pynini.closure(NEMO_NOT_QUOTE) - + pynutil.delete("\"") + + unit_component ) - graph = pynini.closure(sign_component + delete_space) + (cardinal_graph | decimal_graph) + graph_math_cardinal = pynutil.delete("cardinal { ") + cardinal + pynutil.delete(" } ") + + graph_measures = graph_decimal | graph_cardinal | graph_fraction + graph_maths = graph_math_cardinal + + final_graph = graph_maths | graph_measures - self.fst = self.delete_tokens(graph).optimize() + delete_tokens = self.delete_tokens(final_graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/money.py b/nemo_text_processing/text_normalization/zh/verbalizers/money.py index a749018fd..74f517d01 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/money.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/money.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class MoneyFst(GraphFst): """ @@ -30,13 +31,13 @@ def __init__(self, decimal: GraphFst, deterministic: bool = True, lm: bool = Fal super().__init__(name="money", kind="verbalize", deterministic=deterministic) # components to combine to make graphs - number_component = pynutil.delete("integer: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + number_component = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") currency_component = pynutil.delete("currency: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") decimal_component = decimal.decimal_component unit_only_component = ( (pynutil.delete("currency: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) - | (pynutil.delete("currency_major: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) - | (pynutil.delete("currency_minor: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) + | (pynutil.delete("currency_maj: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) + | (pynutil.delete("currency_min: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")) ) # graphs diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/ordinal.py b/nemo_text_processing/text_normalization/zh/verbalizers/ordinal.py index dd9b303c9..d019355e2 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/ordinal.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/ordinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + class OrdinalFst(GraphFst): """ @@ -24,8 +25,8 @@ class OrdinalFst(GraphFst): tokens { ordinal { integer: "第一千万" } } -> 第一千万 """ - def __init__(self): - super().__init__(name="ordinal", kind="verbalize") + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic) symbol = pynini.union("-", "~", "——", "—") dash = pynini.cross(symbol, "到") diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/zh/verbalizers/post_processing.py new file mode 100644 index 000000000..8b196dcaf --- /dev/null +++ b/nemo_text_processing/text_normalization/zh/verbalizers/post_processing.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini + +from nemo_text_processing.text_normalization.en.graph_utils import ( + NEMO_NOT_SPACE, + NEMO_SIGMA, + delete_space, + generator_main, +) +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + """ + Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. + removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + + Args: + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "zh_tn_post_processing.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f'Post processing graph was restored from {far_file}.') + else: + self.set_punct_dict() + self.fst = self.get_punct_postprocess_graph() + + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def set_punct_dict(self): + self.punct_marks = { + "'": [ + "'", + '´', + 'ʹ', + 'ʻ', + 'ʼ', + 'ʽ', + 'ʾ', + 'ˈ', + 'ˊ', + 'ˋ', + '˴', + 'ʹ', + '΄', + '՚', + '՝', + 'י', + '׳', + 'ߴ', + 'ߵ', + 'ᑊ', + 'ᛌ', + '᾽', + '᾿', + '`', + '´', + '῾', + '‘', + '’', + '‛', + '′', + '‵', + 'ꞌ', + '＇', + '｀', + '𖽑', + '𖽒', + ], + } + + def get_punct_postprocess_graph(self): + """ + Returns graph to post process punctuation marks. + + {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. + By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + """ + + remove_space_around_single_quote = pynini.cdrewrite( + delete_space, NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) + ) + # this works if spaces in between (good) + # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA + + graph = remove_space_around_single_quote.optimize() + + return graph diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/postprocessor.py b/nemo_text_processing/text_normalization/zh/verbalizers/postprocessor.py index 67b0b1954..dcdd73622 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/postprocessor.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/postprocessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini +from pynini.lib import pynutil, utf8 + from nemo_text_processing.text_normalization.zh.graph_utils import ( NEMO_ALPHA, NEMO_DIGIT, @@ -21,19 +25,22 @@ GraphFst, ) from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil, utf8 class PostProcessor(GraphFst): ''' - Postprocessing of TN, now contains: - 1. punctuation removal - 2. letter case conversion - 3. oov tagger + Postprocessing of TN, now contains: + 1. punctuation removal + 2. letter case conversion + 3. oov tagger ''' def __init__( - self, remove_puncts: bool = False, to_upper: bool = False, to_lower: bool = False, tag_oov: bool = False, + self, + remove_puncts: bool = False, + to_upper: bool = False, + to_lower: bool = False, + tag_oov: bool = False, ): super().__init__(name="PostProcessor", kind="processor") diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/time.py b/nemo_text_processing/text_normalization/zh/verbalizers/time.py index fd9c41d55..11105a916 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/time.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/time.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,10 @@ import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space from nemo_text_processing.text_normalization.zh.utils import get_abs_path -from pynini.lib import pynutil class TimeFst(GraphFst): @@ -37,9 +38,9 @@ def __init__(self, deterministic: bool = True): alphabet_pm = pynini.string_file(get_abs_path("data/time/PM.tsv")) # fundamental components - hour_component = pynutil.delete("hour: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") - minute_component = pynutil.delete("minute: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") - second_component = pynutil.delete("second: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + hour_component = pynutil.delete("hours: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + minute_component = pynutil.delete("minutes: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") + second_component = pynutil.delete("seconds: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"") graph_regular = ( hour_component | minute_component @@ -51,7 +52,7 @@ def __init__(self, deterministic: bool = True): ) # back count 三点差五分 - delete_verb = pynutil.delete("verb: \"") + pynini.accep("差") + pynutil.delete("\"") + delete_verb = pynutil.delete("morphosyntactic_features: \"") + pynini.accep("差") + pynutil.delete("\"") graph_back_count = ( ( pynini.closure(delete_verb + pynutil.insert(' ')) diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/zh/verbalizers/verbalize.py index 19a398b1a..a927f4716 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/verbalize.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,25 +11,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst + +from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, delete_space from nemo_text_processing.text_normalization.zh.verbalizers.cardinal import CardinalFst from nemo_text_processing.text_normalization.zh.verbalizers.date import DateFst from nemo_text_processing.text_normalization.zh.verbalizers.decimal import DecimalFst from nemo_text_processing.text_normalization.zh.verbalizers.fraction import FractionFst -from nemo_text_processing.text_normalization.zh.verbalizers.math_symbol import MathSymbol -from nemo_text_processing.text_normalization.zh.verbalizers.measure import Measure +from nemo_text_processing.text_normalization.zh.verbalizers.measure import MeasureFst from nemo_text_processing.text_normalization.zh.verbalizers.money import MoneyFst from nemo_text_processing.text_normalization.zh.verbalizers.ordinal import OrdinalFst from nemo_text_processing.text_normalization.zh.verbalizers.time import TimeFst from nemo_text_processing.text_normalization.zh.verbalizers.whitelist import Whitelist -from nemo_text_processing.text_normalization.zh.verbalizers.word import Char +from nemo_text_processing.text_normalization.zh.verbalizers.word import WordFst class VerbalizeFst(GraphFst): """ Composes other verbalizer grammars. - For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. More details to deployment at NeMo/tools/text_processing_deployment. Args: deterministic: if True will provide a single transduction option, @@ -41,28 +43,27 @@ def __init__(self, deterministic: bool = True): date = DateFst(deterministic=deterministic) cardinal = CardinalFst(deterministic=deterministic) - char = Char(deterministic=deterministic) + ordinal = OrdinalFst(deterministic=deterministic) decimal = DecimalFst(deterministic=deterministic) + word = WordFst(deterministic=deterministic) fraction = FractionFst(decimal=decimal, deterministic=deterministic) - math_symbol = MathSymbol(deterministic=deterministic) money = MoneyFst(decimal=decimal, deterministic=deterministic) - measure = Measure(deterministic=deterministic) - ordinal = OrdinalFst() + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) time = TimeFst(deterministic=deterministic) whitelist = Whitelist(deterministic=deterministic) graph = pynini.union( date.fst, cardinal.fst, + ordinal.fst, decimal.fst, fraction.fst, - char.fst, - math_symbol.fst, + word.fst, money.fst, measure.fst, - ordinal.fst, time.fst, whitelist.fst, ) + graph = pynini.closure(delete_space) + graph + pynini.closure(delete_space) self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/zh/verbalizers/verbalize_final.py index 41bc630ed..846254938 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/verbalize_final.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,18 +14,17 @@ import os import pynini +from pynini.lib import pynutil + from nemo_text_processing.text_normalization.zh.graph_utils import GraphFst, delete_space, generator_main from nemo_text_processing.text_normalization.zh.verbalizers.postprocessor import PostProcessor from nemo_text_processing.text_normalization.zh.verbalizers.verbalize import VerbalizeFst -from pynini.lib import pynutil -# import logging +# from nemo.utils import logging class VerbalizeFinalFst(GraphFst): - """ - - """ + """ """ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) @@ -37,13 +36,17 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = pynini.Far(far_file, mode="r")["verbalize"] else: token_graph = VerbalizeFst(deterministic=deterministic) + token_verbalizer = ( pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") ) verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space) - postprocessor = PostProcessor(remove_puncts=False, to_upper=False, to_lower=False, tag_oov=False,) + postprocessor = PostProcessor( + remove_puncts=False, + to_upper=False, + to_lower=False, + tag_oov=False, + ) self.fst = (verbalizer @ postprocessor.fst).optimize() - if far_file: - generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/zh/verbalizers/whitelist.py index e9780744f..0dc6cca68 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/whitelist.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/whitelist.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,14 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + import pynini -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst + class Whitelist(GraphFst): ''' - tokens { whitelist: "ATM" } -> A T M + tokens { whitelist: "ATM" } -> A T M ''' def __init__(self, deterministic: bool = True, lm: bool = False): diff --git a/nemo_text_processing/text_normalization/zh/verbalizers/word.py b/nemo_text_processing/text_normalization/zh/verbalizers/word.py index 5f241f476..b481d78d5 100644 --- a/nemo_text_processing/text_normalization/zh/verbalizers/word.py +++ b/nemo_text_processing/text_normalization/zh/verbalizers/word.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst + + from pynini.lib import pynutil +from nemo_text_processing.text_normalization.zh.graph_utils import NEMO_NOT_QUOTE, GraphFst + -class Char(GraphFst): +class WordFst(GraphFst): ''' - tokens { char: "你" } -> 你 + tokens { char: "你" } -> 你 ''' def __init__(self, deterministic: bool = True, lm: bool = False): diff --git a/nemo_text_processing/utils/__init__.py b/nemo_text_processing/utils/__init__.py new file mode 100644 index 000000000..5680454ae --- /dev/null +++ b/nemo_text_processing/utils/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATE. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.utils.logging import logger diff --git a/nemo_text_processing/utils/logging.py b/nemo_text_processing/utils/logging.py new file mode 100644 index 000000000..96187bfdf --- /dev/null +++ b/nemo_text_processing/utils/logging.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +# overriding with the library specific logger, so that it's possible to +# customize in any downstream applications +logger = logging.getLogger("NeMo-text-processing") +c_handler = logging.StreamHandler() +logger.addHandler(c_handler) +logger.setLevel(logging.INFO) +logFormatter = logging.Formatter(fmt=' %(name)s :: %(levelname)-8s :: %(message)s') +c_handler.setFormatter(logFormatter) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 8c8de986e..6622e2d49 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,10 +3,10 @@ editdistance inflect joblib pandas -pynini==2.1.5 +pynini==2.1.6.post1 regex sacremoses>=0.0.43 -setuptools==65.5.1 +setuptools>=65.5.1 tqdm>=4.41.0 transformers wget diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt index d89ed803a..aacfde319 100644 --- a/requirements/requirements_test.txt +++ b/requirements/requirements_test.txt @@ -1,8 +1,8 @@ -black==19.10b0 -click==8.0.2 -isort[requirements] < 5 +black==25.1.0 +click>=8.0.2 +isort[requirements]>5.1.0,<=6.0.1 parameterized -pynini==2.1.5 +pynini==2.1.6.post1 pytest pytest-runner ruamel.yaml diff --git a/setup.cfg b/setup.cfg index 57d6480e8..5b0dd345a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ markers = [isort] known_localfolder = nemo,tests -sections = FUTURE,STDLIB,THIRDPARTY,LOCALFOLDER +sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER default_section = THIRDPARTY skip = setup.py diff --git a/setup.py b/setup.py index 4667b49e8..e22afbab3 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,9 @@ elif os.path.exists('README.rst'): # codec is used for consistent encoding long_description = codecs.open( - os.path.join(os.path.abspath(os.path.dirname(__file__)), 'README.rst'), 'r', encoding='utf-8', + os.path.join(os.path.abspath(os.path.dirname(__file__)), 'README.rst'), + 'r', + encoding='utf-8', ).read() long_description_content_type = "text/x-rst" @@ -125,7 +127,8 @@ def __call_checker(self, base_command, scope, check): command.extend(['--check', '--diff']) self.announce( - msg='Running command: %s' % str(' '.join(command)), level=distutils_log.INFO, + msg='Running command: %s' % str(' '.join(command)), + level=distutils_log.INFO, ) return_code = subprocess.call(command) @@ -133,10 +136,18 @@ def __call_checker(self, base_command, scope, check): return return_code def _isort(self, scope, check): - return self.__call_checker(base_command=self.__ISORT_BASE.split(), scope=scope, check=check,) + return self.__call_checker( + base_command=self.__ISORT_BASE.split(), + scope=scope, + check=check, + ) def _black(self, scope, check): - return self.__call_checker(base_command=self.__BLACK_BASE.split(), scope=scope, check=check,) + return self.__call_checker( + base_command=self.__BLACK_BASE.split(), + scope=scope, + check=check, + ) def _pass(self): self.announce(msg='\033[32mPASS\x1b[0m', level=distutils_log.INFO) diff --git a/tests/conftest.py b/tests/conftest.py index 77ee095de..a26dab531 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,13 +56,15 @@ def pytest_addoption(parser): help="path to a directory with .far grammars for CPU TN/ITN tests, (DEFAULT: None, i.e. no cache)", ) parser.addoption( - '--run_audio_based', action='store_true', help="pass this argument to run audio-based TN tests", + '--run_audio_based', + action='store_true', + help="pass this argument to run audio-based TN tests", ) @pytest.fixture def device(request): - """ Simple fixture returning string denoting the device [CPU | GPU] """ + """Simple fixture returning string denoting the device [CPU | GPU]""" if request.config.getoption("--cpu"): return "CPU" else: @@ -104,7 +106,7 @@ def cleanup_local_folder(): @pytest.fixture def test_data_dir(): - """ Fixture returns test_data_dir. """ + """Fixture returns test_data_dir.""" # Test dir. test_data_dir_ = join(dirname(__file__), __TEST_DATA_SUBDIR) return test_data_dir_ @@ -148,10 +150,12 @@ def pytest_configure(config): If file absent or sizes not equal, function downloads the archive from github and unpacks it. """ config.addinivalue_line( - "markers", "run_only_on(device): runs the test only on a given device [CPU | GPU]", + "markers", + "run_only_on(device): runs the test only on a given device [CPU | GPU]", ) config.addinivalue_line( - "markers", "with_downloads: runs the test using data present in tests/.data", + "markers", + "with_downloads: runs the test using data present in tests/.data", ) # Test dir and archive filepath. test_dir = join(dirname(__file__), __TEST_DATA_SUBDIR) @@ -218,7 +222,7 @@ def pytest_configure(config): extract_data_from_tar(test_dir, test_data_archive, local_data=config.option.use_local_test_data) # Set cache directory for TN/ITN tests - from .nemo_text_processing.utils import set_cache_dir, set_audio_based_tests + from .nemo_text_processing.utils import set_audio_based_tests, set_cache_dir set_cache_dir(config.option.tn_cache_dir) set_audio_based_tests(config.option.run_audio_based) diff --git a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt index 162e15799..a0694533d 100644 --- a/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/ar/data_text_normalization/test_cases_money.txt @@ -23,4 +23,5 @@ $20.1~عشرون دولار وعشرة سنتات $20.10~عشرون دولار وعشرة سنتات $0.10~عشرة سنتات $0.03~ثلاثة سنتات -$0.20~عشرون سنت \ No newline at end of file +$0.20~عشرون سنت +aed1.2~واحد درهم إماراتي وعشرون \ No newline at end of file diff --git a/tests/nemo_text_processing/ar/test_cardinal.py b/tests/nemo_text_processing/ar/test_cardinal.py index f63890166..85e9c7d37 100644 --- a/tests/nemo_text_processing/ar/test_cardinal.py +++ b/tests/nemo_text_processing/ar/test_cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from tests.nemo_text_processing.utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/ar/test_decimal.py b/tests/nemo_text_processing/ar/test_decimal.py index 50dcb3115..8c4753aee 100644 --- a/tests/nemo_text_processing/ar/test_decimal.py +++ b/tests/nemo_text_processing/ar/test_decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from tests.nemo_text_processing.utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/ar/test_fraction.py b/tests/nemo_text_processing/ar/test_fraction.py index 67f030be6..6ab278fc4 100644 --- a/tests/nemo_text_processing/ar/test_fraction.py +++ b/tests/nemo_text_processing/ar/test_fraction.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from tests.nemo_text_processing.utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/ar/test_money.py b/tests/nemo_text_processing/ar/test_money.py index 219414dc3..2aa49ba9a 100644 --- a/tests/nemo_text_processing/ar/test_money.py +++ b/tests/nemo_text_processing/ar/test_money.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from tests.nemo_text_processing.utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -48,6 +49,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio: pred_non_deterministic = self.normalizer_with_audio.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/ar/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ar/test_sparrowhawk_normalization.sh new file mode 100755 index 000000000..6998a6fbc --- /dev/null +++ b/tests/nemo_text_processing/ar/test_sparrowhawk_normalization.sh @@ -0,0 +1,71 @@ +#! /bin/sh +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests/ar"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + while IFS= read -r testcase; do + IFS='~' read -r written spoken <<< "$testcase" + + escaped_written=$(printf '%s' "$written" | sed 's/\\/\\\\/g') + denorm_pred=$(echo "$escaped_written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +# For test files stored as expected~input (spoken~written). +runtest_swapped () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + while IFS= read -r testcase; do + IFS='~' read -r spoken written <<< "$testcase" + + escaped_written=$(printf '%s' "$written" | sed 's/\\/\\\\/g') + denorm_pred=$(echo "$escaped_written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + input=$TEST_DIR/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNDecimal() { + input=$TEST_DIR/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +testTNFraction() { + input=$TEST_DIR/data_text_normalization/test_cases_fraction.txt + runtest_swapped $input +} + +testTNMeasure() { + input=$TEST_DIR/data_text_normalization/test_cases_measure.txt + runtest_swapped $input +} + +testTNMoney() { + input=$TEST_DIR/data_text_normalization/test_cases_money.txt + runtest $input +} + +# Remove all command-line arguments +shift $# + +# Load shUnit2 +. /workspace/shunit2/shunit2 diff --git a/tests/nemo_text_processing/ar/test_whitelist.py b/tests/nemo_text_processing/ar/test_whitelist.py index d0291703d..29ad1dc6d 100644 --- a/tests/nemo_text_processing/ar/test_whitelist.py +++ b/tests/nemo_text_processing/ar/test_whitelist.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from tests.nemo_text_processing.utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/audio_based_utils/test_audio_based_utils.py b/tests/nemo_text_processing/audio_based_utils/test_audio_based_utils.py index c2c8dbc97..2f9b723ac 100644 --- a/tests/nemo_text_processing/audio_based_utils/test_audio_based_utils.py +++ b/tests/nemo_text_processing/audio_based_utils/test_audio_based_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest + from nemo_text_processing.text_normalization.utils_audio_based import get_alignment diff --git a/tests/nemo_text_processing/de/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/de/data_text_normalization/test_cases_date.txt index 7de5cb7d7..d1b58fe80 100644 --- a/tests/nemo_text_processing/de/data_text_normalization/test_cases_date.txt +++ b/tests/nemo_text_processing/de/data_text_normalization/test_cases_date.txt @@ -3,7 +3,6 @@ vierzehnter januar~14. januar erster januar~1. januar dreißigster juni~30. juni zweiter märz zwei tausend drei~02.03.2003 -zweiter märz zwei tausend drei~02.03.2003 zweiter märz zwei tausend drei~2.3.2003 zweiter märz~2.3 zweiter märz~02.03 diff --git a/tests/nemo_text_processing/de/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/de/data_text_normalization/test_cases_decimal.txt index 551114d8e..e8bde3941 100644 --- a/tests/nemo_text_processing/de/data_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/de/data_text_normalization/test_cases_decimal.txt @@ -4,4 +4,7 @@ vier hundert sechzig millionen~460 millionen ein hundert zwanzig millionen~120 millionen zehn millionen~10 millionen minus sechzig komma zwei vier null null~-60,2400 -acht hundert achtzehn komma drei null drei~818,303 \ No newline at end of file +acht hundert achtzehn komma drei null drei~818,303 +eins , zwei komma drei~1,2,3 +eins komma zwei , drei komma vier~1,2,3,4 +eins , zwei komma drei , vier komma fünf~1,2,3,4,5 \ No newline at end of file diff --git a/tests/nemo_text_processing/de/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/de/data_text_normalization/test_cases_electronic.txt index fc9b0fc66..2c70d8932 100644 --- a/tests/nemo_text_processing/de/data_text_normalization/test_cases_electronic.txt +++ b/tests/nemo_text_processing/de/data_text_normalization/test_cases_electronic.txt @@ -1,4 +1,4 @@ -c d f at a b c punkt e d u~cdf@abc.edu +c d f at a b c punkt edu~cdf@abc.edu a b c at g mail punkt a b c~abc@gmail.abc a b c at a b c punkt com~abc@abc.com a s d f eins zwei drei at a b c punkt com~asdf123@abc.com @@ -6,4 +6,14 @@ a eins b zwei at a b c punkt com~a1b2@abc.com a b drei bindestrich s d d bindestrich drei at g mail punkt com~ab3-sdd-3@gmail.com h t t p s doppelpunkt slash slash w w w punkt a b c punkt com~https://www.abc.com w w w punkt a b c punkt com~www.abc.com -h t t p s doppelpunkt slash slash w w w punkt a b c punkt com slash a b fragezeichen gleichheitszeichen drei bindestrich slash a b s slash eins~https://www.abc.com/ab?=3-/abs/1 \ No newline at end of file +brettspielversand punkt de.~brettspielversand.de. +w w w punkt e n v e e d y a punkt net.~www.enveedya.net. +w w w punkt amazon punkt com punkt de.~www.amazon.com.de. +h t t p s doppelpunkt slash slash w w w punkt a b c punkt com slash a b fragezeichen gleichheitszeichen drei bindestrich slash a b s slash eins~https://www.abc.com/ab?=3-/abs/1 +at j e n s e n~@jensen +at j e n s e n punkt m e~@jensen.me +at w e z y r eins neun acht sechs~@wezyr1986 +zwei-D-Mammogram~2D-Mammogram +zwei-D-Mammogram~2-D-Mammogram +drei-D-Drucker~3D-Drucker +drei-D-Drucker~3-D-Drucker \ No newline at end of file diff --git a/tests/nemo_text_processing/de/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/de/data_text_normalization/test_cases_time.txt index 2b5216d20..ad6c7d7bd 100644 --- a/tests/nemo_text_processing/de/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/de/data_text_normalization/test_cases_time.txt @@ -23,4 +23,6 @@ vier und zwanzig uhr fünf und vierzig~24:45 Uhr vier und zwanzig uhr fünfzehn~24:15 Uhr null uhr null minuten null sekunden~00:00:00 Uhr ein uhr eine minute eine sekunde e s t~01:01:01 Uhr est -zwei uhr zwei minuten drei und zwanzig sekunden~02:02:23 Uhr \ No newline at end of file +zwei uhr zwei minuten drei und zwanzig sekunden~02:02:23 Uhr +zwei uhr dreißig~2.30 Uhr +zwei uhr dreißig~02.30 Uhr \ No newline at end of file diff --git a/tests/nemo_text_processing/de/test_cardinal.py b/tests/nemo_text_processing/de/test_cardinal.py index 1119a72d3..d7fd9a33e 100644 --- a/tests/nemo_text_processing/de/test_cardinal.py +++ b/tests/nemo_text_processing/de/test_cardinal.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_date.py b/tests/nemo_text_processing/de/test_date.py index e4a0cb8cc..13a8785e7 100644 --- a/tests/nemo_text_processing/de/test_date.py +++ b/tests/nemo_text_processing/de/test_date.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_decimal.py b/tests/nemo_text_processing/de/test_decimal.py index 149b8ee75..17cdeadfe 100644 --- a/tests/nemo_text_processing/de/test_decimal.py +++ b/tests/nemo_text_processing/de/test_decimal.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_electronic.py b/tests/nemo_text_processing/de/test_electronic.py index 3400dcc2e..eaf442533 100644 --- a/tests/nemo_text_processing/de/test_electronic.py +++ b/tests/nemo_text_processing/de/test_electronic.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_fraction.py b/tests/nemo_text_processing/de/test_fraction.py index 2e068bc15..9f7b40e97 100644 --- a/tests/nemo_text_processing/de/test_fraction.py +++ b/tests/nemo_text_processing/de/test_fraction.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_measure.py b/tests/nemo_text_processing/de/test_measure.py index c6220664b..27adc13a9 100644 --- a/tests/nemo_text_processing/de/test_measure.py +++ b/tests/nemo_text_processing/de/test_measure.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_money.py b/tests/nemo_text_processing/de/test_money.py index 0989104ad..5124205cf 100644 --- a/tests/nemo_text_processing/de/test_money.py +++ b/tests/nemo_text_processing/de/test_money.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_normalization_with_audio.py b/tests/nemo_text_processing/de/test_normalization_with_audio.py index 2f04090c6..05493e8e3 100644 --- a/tests/nemo_text_processing/de/test_normalization_with_audio.py +++ b/tests/nemo_text_processing/de/test_normalization_with_audio.py @@ -15,9 +15,10 @@ import logging import pytest -from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + from ..utils import CACHE_DIR, get_test_cases_multiple diff --git a/tests/nemo_text_processing/de/test_ordinal.py b/tests/nemo_text_processing/de/test_ordinal.py index a703564e3..e608922e9 100644 --- a/tests/nemo_text_processing/de/test_ordinal.py +++ b/tests/nemo_text_processing/de/test_ordinal.py @@ -14,9 +14,10 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/de/test_sparrowhawk_inverse_text_normalization.sh index d7aba4c02..fa24eb640 100644 --- a/tests/nemo_text_processing/de/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/de/test_sparrowhawk_inverse_text_normalization.sh @@ -2,9 +2,13 @@ PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/"} + runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/de/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/de/test_sparrowhawk_normalization.sh index 3dfdfe9f3..ac40cbae7 100644 --- a/tests/nemo_text_processing/de/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/de/test_sparrowhawk_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/de/test_telephone.py b/tests/nemo_text_processing/de/test_telephone.py index bb3d0ce7a..57595fac8 100644 --- a/tests/nemo_text_processing/de/test_telephone.py +++ b/tests/nemo_text_processing/de/test_telephone.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_time.py b/tests/nemo_text_processing/de/test_time.py index 709c2802c..33768b607 100644 --- a/tests/nemo_text_processing/de/test_time.py +++ b/tests/nemo_text_processing/de/test_time.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_whitelist.py b/tests/nemo_text_processing/de/test_whitelist.py index d740c966b..9db19cac4 100644 --- a/tests/nemo_text_processing/de/test_whitelist.py +++ b/tests/nemo_text_processing/de/test_whitelist.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/de/test_word.py b/tests/nemo_text_processing/de/test_word.py index 0026e0901..259b79bfb 100644 --- a/tests/nemo_text_processing/de/test_word.py +++ b/tests/nemo_text_processing/de/test_word.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_measure.txt index e4d78e413..166bccc81 100644 --- a/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_measure.txt @@ -75,7 +75,7 @@ eight hundred kilo watt hours~800 kWh eight hundred kilowatts~800 kW eight hundred megahertz~800 mhz eight hundred ninety four c c~894 cc -eight hundred ninety kilowatts~890 kW` +eight hundred ninety kilowatts~890 kW eight hundred ninety millimeters~890 mm eight hundred ninety two square kilometers~892 km² eight hundred seventy horsepower~870 hp diff --git a/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_whitelist.txt index cca1dc440..8d62bcc43 100644 --- a/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/en/data_inverse_text_normalization/test_cases_whitelist.txt @@ -7,7 +7,6 @@ s and p five hundred~S&P 500 seven eleven stores~7-eleven stores r t x~RTX cat five e~CAT5e -nvidia a one hundred~Nvidia A100 c u d n n~cuDNN p c i e x eight~PCIe x8 l g a eleven fifty~LGA 1150 diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_address.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_address.txt index bd9394163..08c5f7688 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_address.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_address.txt @@ -7,3 +7,5 @@ 12 S 1st st~twelve South first Street 1990 for the Ata ST~nineteen ninety for the Ata ST Main St.~Main St. +Nancy lived at 1428 Elm St. It was a strange place.~Nancy lived at fourteen twenty eight Elm Street. It was a strange place. +(incorrect normalization) Nancy lived at 1428 Elm St. 4 doors down lived her best friend.~(incorrect normalization) Nancy lived at fourteen twenty eight Elm Street four doors down lived her best friend. \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_electronic.txt index 0e930452a..498528463 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_electronic.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_electronic.txt @@ -4,7 +4,7 @@ a@hotmail.fr~a at hotmail dot FR a@hotmail.it~a at hotmail dot IT a@aol.it~a at aol dot IT a@msn.it~a at msn dot IT -cdf@abc.edu~cdf at abc dot EDU +cdf@abc.edu~cdf at abc dot edu abc@gmail.abc~abc at gmail dot ABC abc@abc.com~abc at abc dot com asdf123@abc.com~asdf one two three at abc dot com @@ -13,9 +13,9 @@ ab3.sdd.3@gmail.com~ab three dot sdd dot three at gmail dot com email is abc1@gmail.com~email is abc one at gmail dot com abs@nvidia.com~abs at NVIDIA dot com email is a-1.b3_c&-d@gma4i-l.com test~email is a dash one dot b three underscore c ampersand dash d at gma four i dash l dot com test -nvidia.com~NVIDIA dot com +nvidia.com~nvidia dot com test.com~test dot com -test.abc~test.abc +test.abc~test dot abc http://www.ourdailynews.com.sm~HTTP colon slash slash WWW dot ourdailynews dot com dot SM https://www.ourdailynews.com.sm~HTTPS colon slash slash WWW dot ourdailynews dot com dot SM www.ourdailynews.com.sm~WWW dot ourdailynews dot com dot SM @@ -24,8 +24,8 @@ sdf@gmail.com.sm~sdf at gmail dot com dot SM sdf@gmail.com/123-sm~sdf at gmail dot com slash one two three dash SM sdf@gmail.abc/123-sm~sdf at gmail dot ABC slash one two three dash SM sdf@gmail.abc/123456-sm~sdf at gmail dot ABC slash one two three four five six dash SM -ourdailynews.com/12-sm~ourdailynews dot com slash one two dash SM -ourdailynews.abc~ourdailynews.abc +ourdailynews.com/12-sm~ourdailynews dot com slash one two dash sm +ourdailynews.abc~ourdailynews dot abc file:///c/code/NeMo/docs/build/html/processing/intro.html~file colon slash slash slash c slash code slash nemo slash docs slash build slash html slash processing slash intro dot HTML file:///photos/image.jpg~file colon slash slash slash photos slash image dot jpeg electronic test.com and test2.uk~electronic test dot com and test two dot UK @@ -38,3 +38,8 @@ rtxprohelp@exchange.nvidia.com~RTX pro help at exchange dot NVIDIA dot com enterpriseservices@nvidia.com~enterprise services at NVIDIA dot com enterprise-services@nvidia.com~enterprise dash services at NVIDIA dot com https://www.nvidia.com/dgx-basepod/~HTTPS colon slash slash WWW dot NVIDIA dot com slash DGX dash BASEPOD slash +i can use your card ending in 8876~i can use your card ending in eight eight seven six +upgrade/update~upgrade slash update +upgrade / update~upgrade slash update +upgrade/update/downgrade~upgrade slash update slash downgrade +5.4, or 5.5~five point four, or five point five \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt index 4d16a18ed..c1981829f 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_measure.txt @@ -17,3 +17,5 @@ covid-19.5~covid- nineteen point five 1234-123kg~one thousand two hundred and thirty four to one hundred and twenty three kilograms 45º&C~forty five degree and C 41,459.00 km³~forty one thousand four hundred and fifty nine point zero zero cubic kilometers +Nancy lived on 1428 Elm St.~Nancy lived on fourteen twenty eight Elm Street +Nancy lived on 1428 Elm St. It was a strange place.~Nancy lived on fourteen twenty eight Elm Street. It was a strange place. \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_money.txt index d1218e8bd..1683e7893 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_money.txt @@ -58,8 +58,14 @@ $18129~eighteen thousand one hundred and twenty nine dollars £1.20~one pound twenty pence £1.111~one point one one one pounds £2.01~two pounds one penny -$50.00-$100.00~fifty dollars - one hundred dollars +$50.00-$100.00~fifty dollars to one hundred dollars $1,925.21~one thousand nine hundred and twenty five dollars twenty one cents $1,234.123~one thousand two hundred and thirty four point one two three dollars US $76.3 trillion~US seventy six point three trillion dollars US$76.3 trillion~seventy six point three trillion us dollars +The price for each canned salmon is $5 , each bottle of peanut butter is $3~The price for each canned salmon is five dollars , each bottle of peanut butter is three dollars +$20/mo is what we are currently charging.~twenty dollars per month is what we are currently charging. +$350/yr is the fee.~three hundred and fifty dollars per year is the fee. +£10/wk sounds good to us.~ten pounds per week sounds good to us. +$1/d is the new subscription cost.~one dollar per day is the new subscription cost. +$0.5/hr is the total cost.~fifty cents per hour is the total cost. \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_ordinal.txt index 2e1b5ec7e..d4f073525 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_ordinal.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_ordinal.txt @@ -24,4 +24,4 @@ 21th~twenty one th 121st~one hundred twenty first 111th~one hundred eleventh -111st~one hundred eleven st \ No newline at end of file +111st~one one one st \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt index 5f5b249c1..a014ba9e4 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt @@ -39,7 +39,6 @@ a: ``b``, ``c``, ``d``, ``e``, ``f`` or ``g``.~a: "b", "c", "d", "e", "f" or "g" "And now," he said, "He,~"And now," he said, "He, 80's~eighty's test .b@gmail.com~test. b at gmail dot com -ourdailynews.abc/123-sm~ourdailynews. abc/one hundred twenty three-sm 123,000012~one hundred and twenty three, zero zero zero zero one two 9000,000th~nine thousand, zero zero zero th 16 July, 1943~the sixteenth of july, nineteen forty three @@ -59,6 +58,6 @@ dr. Evil~dr. Evil 1!!!!~one!!!! (1)Hello~(one) Hello ÀÁÂà check §- and ƛ, also ɧ~ÀÁÂà check section - and ƛ, also ɧ -Hi it's 5pm,4A.M.?-34. Hi,no,yes,34! 12,again,4 and NO?17 and $.01,here & there--0.004kg~Hi it's five PM, four AM.? minus thirty four. Hi,no,yes, thirty four! twelve, again, four and NO? seventeen and one cent, here and there - minus zero point zero zero four kilograms +Hi it's 5pm,4A.M.?-34. Hi,no,yes,34! 12,again,4 and NO?17 and $.01,here & there -0.004kg~Hi it's five PM, four AM.? minus thirty four. Hi,no,yes, thirty four! twelve, again, four and NO? seventeen and one cent, here and there minus zero point zero zero four kilograms 1°C.~one degree Celsius. my email is myemail@gmail.com!~my email is myemail at gmail dot com! diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation_match_input.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation_match_input.txt index 51d4aceb5..0e2ea0ed0 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation_match_input.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation_match_input.txt @@ -10,4 +10,4 @@ a: ``b``,``c``, ``d``, ``e``, ``f`` or ``g``.~a: "b","c", "d", "e", "f" or "g". 1 ,~one , (1)Hello~(one)Hello ÀÁÂà check §- and ƛ, also ɧ~ÀÁÂà check section- and ƛ, also ɧ -Hi it's 5pm,4A.M.?-34. Hi,no,yes,34! 12,again,4 and NO?17 and $.01,here & there--0.004kg~Hi it's five PM,four AM.?minus thirty four. Hi,no,yes,thirty four! twelve,again,four and NO?seventeen and one cent,here and there - minus zero point zero zero four kilograms \ No newline at end of file +Hi it's 5pm,4A.M.?-34. Hi,no,yes,34! 12,again,4 and NO?17 and $.01,here & there -0.004kg~Hi it's five PM,four AM.?minus thirty four. Hi,no,yes,thirty four! twelve,again,four and NO?seventeen and one cent,here and there minus zero point zero zero four kilograms \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_range.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_range.txt index 311092806..867cd8df2 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_range.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_range.txt @@ -15,4 +15,6 @@ from 2-10~from two - ten 1960s-1980s~nineteen sixties to nineteen eighties 1960-80~nineteen sixty to eighty mid-1980s~mid nineteen eighties -mid-80s~mid eighties \ No newline at end of file +mid-80s~mid eighties +$250-$300~two hundred and fifty dollars to three hundred dollars +$150.25-$250.67~one hundred and fifty dollars twenty five cents to two hundred and fifty dollars sixty seven cents \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt index 04a03e07e..f142ceb7e 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_serial.txt @@ -10,7 +10,7 @@ MIG-25/235212-asdg~MIG-twenty five/two three five two one two-asdg 1/f-4s~one/f-four s 1/f~one per F 4s~four S -7-eleven~seven-eleven +7-eleven~seven eleven 2x~two x 31/31/100~thirty one/thirty one/one hundred 1-8090~one - eight thousand and ninety @@ -29,3 +29,5 @@ a 4-kilogram bag~a four-kilogram bag 100-car~one hundred-car 123/261788/2021~one hundred twenty three/two six one seven eight eight/two thousand twenty one 2*8~two asterisk eight +my pnr is t2000~my pnr is t two thousand +your otp is ab9453~your otp is ab nine four five three \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_time.txt index e7f1d53e4..36e05d287 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_time.txt @@ -17,3 +17,5 @@ 14:10:30~fourteen hours ten minutes and thirty seconds 2pm-5pm~two PM to five PM 5pm~five PM +21:51:31~twenty one hours fifty one minutes and thirty one seconds +01:01:01~one hour one minute and one second \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt index 7fafb9d77..8ba5bfd9b 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_whitelist.txt @@ -4,3 +4,4 @@ DNA is~DNA is C. S. Lewis~CS Lewis tv~TV and/or~and/or +Our company offers a 401(k) retirement savings plan~Our company offers a four oh one k retirement savings plan \ No newline at end of file diff --git a/tests/nemo_text_processing/en/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/en/data_text_normalization/test_cases_word.txt index c1bd3b58a..b83ad7c31 100644 --- a/tests/nemo_text_processing/en/data_text_normalization/test_cases_word.txt +++ b/tests/nemo_text_processing/en/data_text_normalization/test_cases_word.txt @@ -36,3 +36,4 @@ mar~mar /$€₩£BB¥#%AA and $€₩£¥#%~slash dollar euro won pound BB yen hash percent AA and dollar euro won pound yen hash percent love him while we may,~love him while we may, mar~mar +i saw (22) ducklings~i saw (twenty two) ducklings \ No newline at end of file diff --git a/tests/nemo_text_processing/en/test_address.py b/tests/nemo_text_processing/en/test_address.py index d7d25a3e7..ea8328d10 100644 --- a/tests/nemo_text_processing/en/test_address.py +++ b/tests/nemo_text_processing/en/test_address.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -41,6 +42,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_cardinal.py b/tests/nemo_text_processing/en/test_cardinal.py index a7109414a..f40e0d1f6 100644 --- a/tests/nemo_text_processing/en/test_cardinal.py +++ b/tests/nemo_text_processing/en/test_cardinal.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -62,6 +63,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/en/test_date.py b/tests/nemo_text_processing/en/test_date.py index 390e56e9b..6237d88c5 100644 --- a/tests/nemo_text_processing/en/test_date.py +++ b/tests/nemo_text_processing/en/test_date.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/en/test_decimal.py b/tests/nemo_text_processing/en/test_decimal.py index b5e14647d..ea20f18d6 100644 --- a/tests/nemo_text_processing/en/test_decimal.py +++ b/tests/nemo_text_processing/en/test_decimal.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -60,6 +61,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_electronic.py b/tests/nemo_text_processing/en/test_electronic.py index 5f14e0e9e..4dfec585e 100644 --- a/tests/nemo_text_processing/en/test_electronic.py +++ b/tests/nemo_text_processing/en/test_electronic.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -59,6 +60,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=100, punct_post_process=False, + test_input, + n_tagged=100, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_fraction.py b/tests/nemo_text_processing/en/test_fraction.py index ea7db2cf7..a6186aabb 100644 --- a/tests/nemo_text_processing/en/test_fraction.py +++ b/tests/nemo_text_processing/en/test_fraction.py @@ -14,9 +14,10 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -38,6 +39,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_math.py b/tests/nemo_text_processing/en/test_math.py index 830800145..22859f596 100644 --- a/tests/nemo_text_processing/en/test_math.py +++ b/tests/nemo_text_processing/en/test_math.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -38,6 +39,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_measure.py b/tests/nemo_text_processing/en/test_measure.py index a17881c41..6ea9a0eda 100644 --- a/tests/nemo_text_processing/en/test_measure.py +++ b/tests/nemo_text_processing/en/test_measure.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -60,6 +61,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_money.py b/tests/nemo_text_processing/en/test_money.py index 81a8c99a7..103223d5e 100644 --- a/tests/nemo_text_processing/en/test_money.py +++ b/tests/nemo_text_processing/en/test_money.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -44,7 +45,9 @@ def test_denorm(self, test_input, expected): pred = self.inverse_normalizer_en_cased.inverse_normalize(test_input, verbose=False) assert pred == expected, f"input: {test_input}" - normalizer_en = Normalizer(input_case='cased', lang='en', cache_dir=CACHE_DIR, overwrite_cache=False) + normalizer_en = Normalizer( + input_case='cased', lang='en', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) normalizer_with_audio_en = ( NormalizerWithAudio(input_case='cased', lang='en', cache_dir=CACHE_DIR, overwrite_cache=False) if RUN_AUDIO_BASED_TESTS @@ -60,6 +63,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_normalization_with_audio.py b/tests/nemo_text_processing/en/test_normalization_with_audio.py index 2601d52a2..671da172a 100644 --- a/tests/nemo_text_processing/en/test_normalization_with_audio.py +++ b/tests/nemo_text_processing/en/test_normalization_with_audio.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + from ..utils import CACHE_DIR, get_test_cases_multiple diff --git a/tests/nemo_text_processing/en/test_ordinal.py b/tests/nemo_text_processing/en/test_ordinal.py index 3dc06a19a..dac56bf38 100644 --- a/tests/nemo_text_processing/en/test_ordinal.py +++ b/tests/nemo_text_processing/en/test_ordinal.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -60,6 +61,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_punctuation.py b/tests/nemo_text_processing/en/test_punctuation.py index 941db694d..761b3c9f4 100644 --- a/tests/nemo_text_processing/en/test_punctuation.py +++ b/tests/nemo_text_processing/en/test_punctuation.py @@ -13,15 +13,20 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestPunctuation: normalizer_en = Normalizer( - input_case='cased', lang='en', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True, + input_case='cased', + lang='en', + cache_dir=CACHE_DIR, + overwrite_cache=False, + post_process=True, ) # address is tagged by the measure class diff --git a/tests/nemo_text_processing/en/test_range.py b/tests/nemo_text_processing/en/test_range.py index 3bdb7969f..64b47d898 100644 --- a/tests/nemo_text_processing/en/test_range.py +++ b/tests/nemo_text_processing/en/test_range.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -38,6 +39,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_roman.py b/tests/nemo_text_processing/en/test_roman.py index 2f7149ff9..3ef655c65 100644 --- a/tests/nemo_text_processing/en/test_roman.py +++ b/tests/nemo_text_processing/en/test_roman.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -39,6 +40,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=False, + test_input, + n_tagged=30, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/en/test_serial.py b/tests/nemo_text_processing/en/test_serial.py index 4ff0ed982..2a27b1f54 100644 --- a/tests/nemo_text_processing/en/test_serial.py +++ b/tests/nemo_text_processing/en/test_serial.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -37,6 +38,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio_en: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=-1, punct_post_process=False, + test_input, + n_tagged=-1, + punct_post_process=False, ) assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization.sh index 2633670c0..610cd6c20 100644 --- a/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do @@ -21,59 +23,63 @@ runtest () { } testITNCardinal() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_cardinal.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_cardinal.txt runtest $input } testITNDate() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_date.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_date.txt runtest $input } testITNDecimal() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_decimal.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_decimal.txt runtest $input } testITNElectronic() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_electronic.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_electronic.txt runtest $input } testITNOrdinal() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_ordinal.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_ordinal.txt runtest $input } testITNTime() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_time.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_time.txt runtest $input } testITNMeasure() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_measure.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_measure.txt runtest $input } testITNMoney() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_money.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_money.txt runtest $input } testITNWhitelist() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_whitelist.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_whitelist.txt runtest $input } testITNTelephone() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_telephone.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_telephone.txt runtest $input } testITNWord() { - input=$PROJECT_DIR/en/data_inverse_text_normalization/test_cases_word.txt + input=$TEST_DIR/data_inverse_text_normalization/test_cases_word.txt runtest $input } + +# Remove all command-line arguments +shift $# + # Load shUnit2 -. $PROJECT_DIR/../shunit2/shunit2 +. /workspace/shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization_cased.sh b/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization_cased.sh index a7d926357..fe622bbe7 100644 --- a/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization_cased.sh +++ b/tests/nemo_text_processing/en/test_sparrowhawk_inverse_text_normalization_cased.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do @@ -21,59 +23,63 @@ runtest () { } testITNCardinal() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_cardinal.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_cardinal_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_cardinal_cased.txt } testITNDate() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_date.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_date_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_date.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_date_cased.txt } testITNDecimal() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_decimal.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_decimal_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_decimal.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_decimal_cased.txt } testITNElectronic() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_electronic.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_electronic_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_electronic.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_electronic_cased.txt } testITNOrdinal() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_ordinal.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_ordinal_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_ordinal_cased.txt } testITNTime() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_time.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_time_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_time.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_time_cased.txt } testITNMeasure() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_measure.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_measure_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_measure.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_measure_cased.txt } testITNMoney() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_money.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_money_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_money.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_money_cased.txt } testITNWhitelist() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_whitelist.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_whitelist_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_whitelist_cased.txt } testITNTelephone() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_telephone.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_telephone_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_telephone.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_telephone_cased.txt } testITNWord() { - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_word.txt - runtest $PROJECT_DIR/en/data_inverse_text_normalization/test_cases_word_cased.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_word.txt + runtest $TEST_DIR/data_inverse_text_normalization/test_cases_word_cased.txt } + +# Remove all command-line arguments +shift $# + # Load shUnit2 -. $PROJECT_DIR/../shunit2/shunit2 +. /workspace/shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/en/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/en/test_sparrowhawk_normalization.sh index 7baa7c198..f15f2290d 100644 --- a/tests/nemo_text_processing/en/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/en/test_sparrowhawk_normalization.sh @@ -1,16 +1,20 @@ #! /bin/sh - -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do IFS='~' read written spoken <<< $testcase # replace non breaking space with breaking space + # Use below if postprocessor is not used. Comment if it is used denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + # Use below if postprocessor is used. Comment if it is not used + #denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration_pp.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') # trim white space spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" @@ -22,94 +26,97 @@ runtest () { } testTNSpecialText() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_special_text.txt + input=$TEST_DIR/data_text_normalization/test_cases_special_text.txt runtest $input } testTNCardinal() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_cardinal.txt + input=$TEST_DIR/data_text_normalization/test_cases_cardinal.txt runtest $input } testTNDate() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_date.txt + input=$TEST_DIR/data_text_normalization/test_cases_date.txt runtest $input } testTNDecimal() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_decimal.txt + input=$TEST_DIR/data_text_normalization/test_cases_decimal.txt runtest $input } testTNRange() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_range.txt + input=$TEST_DIR/data_text_normalization/test_cases_range.txt runtest $input } testTNSerial() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_serial.txt + input=$TEST_DIR/data_text_normalization/test_cases_serial.txt runtest $input } #testTNRoman() { -# input=$PROJECT_DIR/en/data_text_normalization/test_cases_roman.txt +# input=$TEST_DIR/data_text_normalization/test_cases_roman.txt # runtest $input #} testTNElectronic() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_electronic.txt + input=$TEST_DIR/data_text_normalization/test_cases_electronic.txt runtest $input } testTNFraction() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_fraction.txt + input=$TEST_DIR/data_text_normalization/test_cases_fraction.txt runtest $input } testTNMoney() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_money.txt + input=$TEST_DIR/data_text_normalization/test_cases_money.txt runtest $input } testTNOrdinal() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_ordinal.txt + input=$TEST_DIR/data_text_normalization/test_cases_ordinal.txt runtest $input } testTNTelephone() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_telephone.txt + input=$TEST_DIR/data_text_normalization/test_cases_telephone.txt runtest $input } testTNTime() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_time.txt + input=$TEST_DIR/data_text_normalization/test_cases_time.txt runtest $input } testTNMeasure() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_measure.txt + input=$TEST_DIR/data_text_normalization/test_cases_measure.txt runtest $input } testTNWhitelist() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_whitelist.txt + input=$TEST_DIR/data_text_normalization/test_cases_whitelist.txt runtest $input } testTNWord() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_word.txt + input=$TEST_DIR/data_text_normalization/test_cases_word.txt runtest $input } testTNAddress() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_address.txt + input=$TEST_DIR/data_text_normalization/test_cases_address.txt runtest $input } testTNMath() { - input=$PROJECT_DIR/en/data_text_normalization/test_cases_math.txt + input=$TEST_DIR/data_text_normalization/test_cases_math.txt runtest $input } +# Remove all command-line arguments +shift $# + # Load shUnit2 -. $PROJECT_DIR/../shunit2/shunit2 +. /workspace/shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/en/test_special_text.py b/tests/nemo_text_processing/en/test_special_text.py index f0ba6bb6f..73be5d382 100644 --- a/tests/nemo_text_processing/en/test_special_text.py +++ b/tests/nemo_text_processing/en/test_special_text.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file @@ -40,6 +41,8 @@ def test_norm(self, test_input, expected): # Audio-based normalization will output only options without digits if self.normalizer_with_audio_en and sum([1 for ch in expected if ch.isdigit()]) == 0: pred_non_deterministic = self.normalizer_with_audio_en.normalize( - test_input, n_tagged=30, punct_post_process=True, + test_input, + n_tagged=30, + punct_post_process=True, ) assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/en/test_telephone.py b/tests/nemo_text_processing/en/test_telephone.py index 765660dd3..253abd4d3 100644 --- a/tests/nemo_text_processing/en/test_telephone.py +++ b/tests/nemo_text_processing/en/test_telephone.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/en/test_text_split.py b/tests/nemo_text_processing/en/test_text_split.py index b13a49119..3bbb9aa49 100644 --- a/tests/nemo_text_processing/en/test_text_split.py +++ b/tests/nemo_text_processing/en/test_text_split.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest + from nemo_text_processing.text_normalization.normalize import Normalizer from ..utils import CACHE_DIR diff --git a/tests/nemo_text_processing/en/test_time.py b/tests/nemo_text_processing/en/test_time.py index 4405a65ce..6729b39e9 100644 --- a/tests/nemo_text_processing/en/test_time.py +++ b/tests/nemo_text_processing/en/test_time.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/en/test_whitelist.py b/tests/nemo_text_processing/en/test_whitelist.py index 0cb72082e..ce955807a 100644 --- a/tests/nemo_text_processing/en/test_whitelist.py +++ b/tests/nemo_text_processing/en/test_whitelist.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/en/test_word.py b/tests/nemo_text_processing/en/test_word.py index 188cc818e..41587addc 100644 --- a/tests/nemo_text_processing/en/test_word.py +++ b/tests/nemo_text_processing/en/test_word.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_cardinal_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_cardinal_cased.txt new file mode 100644 index 000000000..15514ae12 --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_cardinal_cased.txt @@ -0,0 +1,30 @@ +Doscientos cincuenta y uno~251 +Novecientos noventa y nueve millones novecientos noventa y nueve mil novecientos noventa y nueve~999999999 +Cero~Cero +Uno~Uno +una~una +dos~dos +Nueve~Nueve +Diez~10 +, uno~, uno +, diez~, 10 +Menos veintitrés~-23 +cien~100 +ciento uno~101 +ciento un~101 +ciento una~101 +mil y uno~1001 +Mil una~1001 +nueve billones setecientos ochenta y nueve mil trescientos ochenta y dos millones quinientos treinta y seis mil ciento treinta~9789382536130 +Doscientos cincuenta y cuatro~254 +ciento cuarenta y siete mil cuatrocientos cincuenta y uno~147451 +Un Millón ciento cincuenta y seis mil ciento setenta y tres~1156173 +Mil quinientos noventa y tres millones setenta y dos mil novecientos sesenta y uno~1593072961 +noventa y siete mil ochocientos ocho billones doscientos sesenta y cuatro mil setecientos setenta y dos millones setecientos noventa y dos mil cinco~97808264772792005 +diecisiete mil ochocientos cincuenta y cinco trillones treinta y seis mil seiscientos cincuenta y siete billones siete mil quinientos noventa y seis millones ciento diez mil novecientos cuarenta y nueve~17855036657007596110949 +diez mil diez billones diez millones cien mil diez~10010000010100010 +Menos veinticinco mil treinta y siete~-25037 +mil doscientos sesenta y cuatro billones trescientos un mil novecientos treinta y ocho millones ciento cuatro~1264301938000104 +menos sesenta~-60 +cuarenta y seis mil seiscientos sesenta y cuatro~46664 +sesenta~60 \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_date_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_date_cased.txt new file mode 100644 index 000000000..98bfd6fe3 --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_date_cased.txt @@ -0,0 +1,8 @@ +Primero De Enero~1 de Enero +Uno de enero~1 de Enero +el uno de Diciembre~el 1 de Diciembre +El primero de diciembre~El 1 de diciembre +Domingo Veintiséis De Octubre~Domingo 26 de Octubre +treinta y uno de diciembre de mil novecientos noventa y dos~31 de diciembre de 1992 +Siglo diecinueve~Siglo xix +doscientos tres antes de Cristo~203 A. C. \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_decimal_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_decimal_cased.txt new file mode 100644 index 000000000..81a91bb3a --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_decimal_cased.txt @@ -0,0 +1,6 @@ +Uno coma dos seis~1,26 +Menos uno coma dos seis~-1,26 +Uno Coma Veintiséis~1,26 +Cero coma Dos seis~0,26 +cero coma veintiséis~0,26 +tres coma ciento cuarenta y uno~3,141 \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_electronic_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_electronic_cased.txt new file mode 100644 index 000000000..2d3f26b9d --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_electronic_cased.txt @@ -0,0 +1,5 @@ +A punto B C Arroba G mail punto com~A.BC@gmail.com +c d f Arroba a b c Punto e d u~cdf@abc.edu +W W W Punto N vidia Punto com~www.nvidia.com +Doble ve doble ve doble ve punto a b c punto es barra e f g~www.abc.es/efg +Doble Ve Doble Ve Doble Ve Punto a b c Punto e s~www.abc.es \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure.txt index 02895142e..6b80918b5 100644 --- a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure.txt @@ -13,7 +13,7 @@ cuatro segundos~4 s cinco litros~5 l tres metros cúbicos~3 m³ dos kilómetros por hora~2 kph -diez grados farenheit~10 ° f +diez grados farenheit~10 ° F dos metros y medio~2 1/2 m tres quintos de metro~3/5 m menos tres y medio metros por hora~-3 1/2 m/h diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure_cased.txt new file mode 100644 index 000000000..ad28add7a --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_measure_cased.txt @@ -0,0 +1,11 @@ +Doscientos metros~200 m +tres horas~3 h +una hora~1 h +Doscientos cuarenta y cinco Millas Por Hora~245 mph +Dos Kilos~2 kg +sesenta coma dos cuatro cero cero kilogramos~60,2400 kg +Menos sesenta coma veinticuatro cero cero kilogramos~-60,2400 kg +menos Ocho Coma Cinco Dos por ciento~-8,52 % +uno Porciento~1 % +tres centímetros~3 cm +dos más dos es igual a cuatro~2 + 2 = 4 \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_money_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_money_cased.txt new file mode 100644 index 000000000..a57e6065a --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_money_cased.txt @@ -0,0 +1,6 @@ +doce dólares y cinco centavos~$12,05 +Doce Dólares Y Cinco Céntimos~$12,05 +setenta y cinco Dólares sesenta y tres~$75,63 +Veintinueve dólares cincuenta centavos~$29,50 +Catorce millones quinientos mil Pesos mexicanos~Mex$14500000 +diez pesos Mexicanos~Mex$10 \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_ordinal_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_ordinal_cased.txt new file mode 100644 index 000000000..0dd13fd54 --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_ordinal_cased.txt @@ -0,0 +1,11 @@ +primero~primero +Tercera~Tercera +Primer~Primer +tercer~tercer +Décima~10.ª +undécimo~11.º +Decimoprimer~11.ᵉʳ +Décimo primer~11.ᵉʳ +Décima Primera~11.ª +(technically ungrammatical) décimo primera~(technically ungrammatical) 11.ª +decimotercero~13.º \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_telephone_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_telephone_cased.txt new file mode 100644 index 000000000..068867d68 --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_telephone_cased.txt @@ -0,0 +1,6 @@ +Uno dos tres uno dos tres cinco seis siete ocho~123-123-5678 +uno veintitrés uno veintitrés cincuenta y seis setenta y ocho~123-123-5678 +Uno Dos Tres Cuatro Cinco Seis Siete Ocho Nueve~123-456-789 +Triple tres uno dos tres cinco seis siete ocho~333-123-5678 +Más uno uno dos tres uno dos tres cinco seis siete ocho~+1-123-123-5678 +más cincuenta y cuatro uno dos tres uno dos tres cinco seis siete ocho Extensión doce~+54-123-123-5678 ext. 12 \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time.txt index 8ea4b35f1..e74a63fda 100644 --- a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time.txt @@ -22,4 +22,4 @@ cuarto para las cero~las 23:45 cuarto para las veinticuatro~las 23:45 diez para las doce~las 11:50 dos y media de la tarde~2:30 p.m. -la una de la tarde u t c más cuatro~la 1:00 p.m. utc+4 +la una de la tarde u t c más cuatro~la 1:00 p.m. UTC+4 diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time_cased.txt new file mode 100644 index 000000000..ba450d79b --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_time_cased.txt @@ -0,0 +1,9 @@ +las dieciséis cincuenta~las 16:50 +la una~la una +Las dos~Las dos +Las tres personas~Las tres personas +Las Dos a eme~Las 2:00 a.m. +la una Pe Eme~la 1:00 P.M. +la una y diez~la 1:10 +la una y Diez a eme~la 1:10 a.m. +La Una Y Diez pe eme~La 1:10 p.m. \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_whitelist.txt index f142f8954..d6aa3211c 100644 --- a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_whitelist.txt @@ -1,5 +1,5 @@ -usted~ud. -ustedes~uds. -habla usted español~habla ud. español -hablan ustedes español~hablan uds. español -estados unidos~ee. uu. \ No newline at end of file +usted~Ud. +ustedes~Uds. +habla usted español~habla Ud. español +hablan ustedes español~hablan Uds. español +estados unidos~EE. UU. \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_word_cased.txt b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_word_cased.txt new file mode 100644 index 000000000..38681016d --- /dev/null +++ b/tests/nemo_text_processing/es/data_inverse_text_normalization/test_cases_word_cased.txt @@ -0,0 +1,11 @@ +~ +Yahoo!~Yahoo! +Veinte!~20 ! +X ~X +—~— +AAA~AAA +Aabach~Aabach +aabenraa~aabenraa +Aachen's~Aachen's +aadri~aadri +aaliyan's~aaliyan's \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/es/data_text_normalization/test_cases_electronic.txt index 398c266ad..39f68ed85 100644 --- a/tests/nemo_text_processing/es/data_text_normalization/test_cases_electronic.txt +++ b/tests/nemo_text_processing/es/data_text_normalization/test_cases_electronic.txt @@ -12,4 +12,11 @@ www.abc.es~w w w punto a b c punto e s http://www.ourdailynews.com.sm~h t t p dos puntos barra barra w w w punto o u r d a i l y n e w s punto com punto s m nvidia.com/1|2~nvidia punto com barra uno barra vertical dos nvidia.com/2^~nvidia punto com barra dos acento circunflejo -www.unicef.org~w w w punto u n i c e f punto o r g \ No newline at end of file +www.unicef.org~w w w punto u n i c e f punto o r g +brettspielversand.de.~b r e t t s p i e l v e r s a n d punto d e . +www.enveedya.net.~w w w punto e n v e e d y a punto net . +www.amazon.com.de.~w w w punto a m a z o n punto com punto d e . +https://www.abc.com~h t t p s dos puntos barra barra w w w punto a b c punto com +@jensen~arroba j e n s e n +@jensen.me~arroba j e n s e n punto m e +@wezyr1986~arroba w e z y r uno nueve ocho seis \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/es/data_text_normalization/test_cases_ordinal.txt index b39ccf4b8..82dd7a63a 100644 --- a/tests/nemo_text_processing/es/data_text_normalization/test_cases_ordinal.txt +++ b/tests/nemo_text_processing/es/data_text_normalization/test_cases_ordinal.txt @@ -113,4 +113,8 @@ 1ro~primero 1ra~primera maría vii~maría séptima~maría séptimo -todo mi reconocimiento~todo mi reconocimiento \ No newline at end of file +todo mi reconocimiento~todo mi reconocimiento +V~quinto +El texto de Li Qin en este libro ahora está disponible en forma de libro electrónico.~El texto de Li Qin en este libro ahora está disponible en forma de libro electrónico. +Xi Jinping es el actual presidente de China.~Xi Jinping es el actual presidente de China. +Matías fue el XI apóstol.~Matías fue el undécimo apóstol. \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/es/data_text_normalization/test_cases_telephone.txt index 406a6289a..cbdc5b2c6 100644 --- a/tests/nemo_text_processing/es/data_text_normalization/test_cases_telephone.txt +++ b/tests/nemo_text_processing/es/data_text_normalization/test_cases_telephone.txt @@ -7,9 +7,9 @@ +1 (123) 123-5678~más uno uno dos tres uno dos tres cinco seis siete ocho 1-800-go-u-haul~uno ochocientos go u haul llámame al 123-123-5678~llámame al uno dos tres uno dos tres cinco seis siete ocho -123.123.0.40~doce tres punto uno dos tres punto cero punto cuatro cero~uno dos tres punto uno dos tres punto cero punto cuatro cero la dirección de ip es 123.123.0.40~la dirección de i p es uno dos tres punto uno dos tres punto cero punto cuatro cero 0-800-hermano~cero ochocientos hermano +58 (123) 123-5678 ext. 12~más cincuenta y ocho uno dos tres uno dos tres cinco seis siete ocho extensión uno dos +58 (123) 123-5678-12~más cincuenta y ocho uno dos tres uno dos tres cinco seis siete ocho extensión uno dos -+54 911 1234-5678~más cincuenta y cuatro nueve uno uno uno dos tres cuatro cinco seis siete ocho \ No newline at end of file ++54 911 1234-5678~más cincuenta y cuatro nueve uno uno uno dos tres cuatro cinco seis siete ocho +123.123.0.40~doce tres punto uno dos tres punto cero punto cuatro cero~uno dos tres punto uno dos tres punto cero punto cuatro cero \ No newline at end of file diff --git a/tests/nemo_text_processing/es/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/es/data_text_normalization/test_cases_whitelist.txt index b091080ca..051ba2a89 100644 --- a/tests/nemo_text_processing/es/data_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/es/data_text_normalization/test_cases_whitelist.txt @@ -2,6 +2,6 @@ el dr.~el doctor sr. rodriguez~señor rodriguez 182 esq. toledo~ciento ochenta y dos esquina toledo da. ana pérez~doña ana pérez -ee. uu.~estados unidos pza.~plaza -c.º b.~camino bajo \ No newline at end of file +c.º b.~camino bajo +ee. uu.~estados unidos \ No newline at end of file diff --git a/tests/nemo_text_processing/es/test_cardinal.py b/tests/nemo_text_processing/es/test_cardinal.py index 748853db5..e1b57fca3 100644 --- a/tests/nemo_text_processing/es/test_cardinal.py +++ b/tests/nemo_text_processing/es/test_cardinal.py @@ -13,23 +13,38 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestCardinal: - inverse_normalizer = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + # inverse_normalizer = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_cardinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): - pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) + assert pred == expected + + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_cardinal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) assert pred == expected normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) diff --git a/tests/nemo_text_processing/es/test_date.py b/tests/nemo_text_processing/es/test_date.py index 150fc23ed..4b12236ee 100644 --- a/tests/nemo_text_processing/es/test_date.py +++ b/tests/nemo_text_processing/es/test_date.py @@ -13,16 +13,20 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestDate: inverse_normalizer = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_date.txt')) @pytest.mark.run_only_on('CPU') @@ -31,6 +35,16 @@ def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_cardinal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = NormalizerWithAudio( diff --git a/tests/nemo_text_processing/es/test_decimal.py b/tests/nemo_text_processing/es/test_decimal.py index 4e9585011..7467e45b2 100644 --- a/tests/nemo_text_processing/es/test_decimal.py +++ b/tests/nemo_text_processing/es/test_decimal.py @@ -13,16 +13,20 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestDecimal: inverse_normalizer = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_decimal.txt')) @pytest.mark.run_only_on('CPU') @@ -31,6 +35,16 @@ def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_decimal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( diff --git a/tests/nemo_text_processing/es/test_electronic.py b/tests/nemo_text_processing/es/test_electronic.py index 7726e0d9f..ae0e4530c 100644 --- a/tests/nemo_text_processing/es/test_electronic.py +++ b/tests/nemo_text_processing/es/test_electronic.py @@ -13,16 +13,20 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestElectronic: inverse_normalizer_es = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_electronic.txt')) @pytest.mark.run_only_on('CPU') @@ -31,6 +35,16 @@ def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_electronic_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( diff --git a/tests/nemo_text_processing/es/test_fraction.py b/tests/nemo_text_processing/es/test_fraction.py index c0022c377..d0e818726 100644 --- a/tests/nemo_text_processing/es/test_fraction.py +++ b/tests/nemo_text_processing/es/test_fraction.py @@ -14,14 +14,31 @@ import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestFraction: + inverse_normalizer = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( diff --git a/tests/nemo_text_processing/es/test_measure.py b/tests/nemo_text_processing/es/test_measure.py index 4474ea16e..572c88d03 100644 --- a/tests/nemo_text_processing/es/test_measure.py +++ b/tests/nemo_text_processing/es/test_measure.py @@ -14,16 +14,20 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestMeasure: inverse_normalizer_es = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_measure.txt')) @pytest.mark.run_only_on('CPU') @@ -32,6 +36,16 @@ def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_measure_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( diff --git a/tests/nemo_text_processing/es/test_money.py b/tests/nemo_text_processing/es/test_money.py index 25f34d810..acc1fea82 100644 --- a/tests/nemo_text_processing/es/test_money.py +++ b/tests/nemo_text_processing/es/test_money.py @@ -14,16 +14,20 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestMoney: inverse_normalizer = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_money.txt')) @pytest.mark.run_only_on('CPU') @@ -32,6 +36,16 @@ def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_money_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( diff --git a/tests/nemo_text_processing/es/test_normalization_with_audio.py b/tests/nemo_text_processing/es/test_normalization_with_audio.py index f16f36084..665738733 100644 --- a/tests/nemo_text_processing/es/test_normalization_with_audio.py +++ b/tests/nemo_text_processing/es/test_normalization_with_audio.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + from ..utils import CACHE_DIR, get_test_cases_multiple diff --git a/tests/nemo_text_processing/es/test_ordinal.py b/tests/nemo_text_processing/es/test_ordinal.py index 5d98c3512..1a48d6da8 100644 --- a/tests/nemo_text_processing/es/test_ordinal.py +++ b/tests/nemo_text_processing/es/test_ordinal.py @@ -14,16 +14,20 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestOrdinal: inverse_normalizer = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_ordinal.txt')) @pytest.mark.run_only_on('CPU') @@ -32,6 +36,16 @@ def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_ordinal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( NormalizerWithAudio(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) @@ -48,6 +62,8 @@ def test_norm(self, test_input, expected): if self.normalizer_with_audio: pred_non_deterministic = self.normalizer_with_audio.normalize( - test_input, n_tagged=500, punct_post_process=False, + test_input, + n_tagged=500, + punct_post_process=False, ) assert expected in pred_non_deterministic diff --git a/tests/nemo_text_processing/es/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/es/test_sparrowhawk_inverse_text_normalization.sh index f6a71c2cb..99eca7a88 100644 --- a/tests/nemo_text_processing/es/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/es/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/es/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/es/test_sparrowhawk_normalization.sh index f3cb17fff..102d7e04f 100644 --- a/tests/nemo_text_processing/es/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/es/test_sparrowhawk_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/es/test_telephone.py b/tests/nemo_text_processing/es/test_telephone.py index 5a0159158..265f877f6 100644 --- a/tests/nemo_text_processing/es/test_telephone.py +++ b/tests/nemo_text_processing/es/test_telephone.py @@ -14,16 +14,20 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestTelephone: inverse_normalizer_es = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_telephone.txt')) @pytest.mark.run_only_on('CPU') @@ -32,6 +36,16 @@ def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_telephone_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( diff --git a/tests/nemo_text_processing/es/test_time.py b/tests/nemo_text_processing/es/test_time.py index 86e1c9893..8b958c508 100644 --- a/tests/nemo_text_processing/es/test_time.py +++ b/tests/nemo_text_processing/es/test_time.py @@ -13,16 +13,20 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestTime: inverse_normalizer_es = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_time.txt')) @pytest.mark.run_only_on('CPU') @@ -31,6 +35,16 @@ def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_time_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio = ( diff --git a/tests/nemo_text_processing/es/test_whitelist.py b/tests/nemo_text_processing/es/test_whitelist.py index f5a58d858..52f27da86 100644 --- a/tests/nemo_text_processing/es/test_whitelist.py +++ b/tests/nemo_text_processing/es/test_whitelist.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/es/test_word.py b/tests/nemo_text_processing/es/test_word.py index 87ab24070..11002ea99 100644 --- a/tests/nemo_text_processing/es/test_word.py +++ b/tests/nemo_text_processing/es/test_word.py @@ -13,16 +13,20 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestWord: inverse_normalizer_es = InverseNormalizer(lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_es_cased = InverseNormalizer( + lang='es', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_word.txt')) @pytest.mark.run_only_on('CPU') @@ -31,6 +35,16 @@ def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) assert pred == expected + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('es/data_inverse_text_normalization/test_cases_word_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_es_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer_es = Normalizer(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) normalizer_with_audio_es = ( NormalizerWithAudio(input_case='cased', lang='es', cache_dir=CACHE_DIR, overwrite_cache=False) diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_cardinal_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_cardinal_cased.txt new file mode 100644 index 000000000..d6fc7a85e --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_cardinal_cased.txt @@ -0,0 +1,60 @@ +Doscientos cincuenta y uno~251 +Novecientos noventa y nueve millones novecientos noventa y nueve mil novecientos noventa y nueve~999999999 +Cero~Cero +Uno~Uno +una~una +dos~dos +Nueve~Nueve +Diez~10 +, uno~, uno +, diez~, 10 +Menos veintitrés~-23 +cien~100 +ciento uno~101 +ciento un~101 +ciento una~101 +mil y uno~1001 +Mil una~1001 +nueve billones setecientos ochenta y nueve mil trescientos ochenta y dos millones quinientos treinta y seis mil ciento treinta~9789382536130 +Doscientos cincuenta y cuatro~254 +ciento cuarenta y siete mil cuatrocientos cincuenta y uno~147451 +Un Millón ciento cincuenta y seis mil ciento setenta y tres~1156173 +Mil quinientos noventa y tres millones setenta y dos mil novecientos sesenta y uno~1593072961 +noventa y siete mil ochocientos ocho billones doscientos sesenta y cuatro mil setecientos setenta y dos millones setecientos noventa y dos mil cinco~97808264772792005 +diecisiete mil ochocientos cincuenta y cinco trillones treinta y seis mil seiscientos cincuenta y siete billones siete mil quinientos noventa y seis millones ciento diez mil novecientos cuarenta y nueve~17855036657007596110949 +diez mil diez billones diez millones cien mil diez~10010000010100010 +Menos veinticinco mil treinta y siete~-25037 +mil doscientos sesenta y cuatro billones trescientos un mil novecientos treinta y ocho millones ciento cuatro~1264301938000104 +menos sesenta~-60 +cuarenta y seis mil seiscientos sesenta y cuatro~46664 +sesenta~60 +Nine trillion seven hundred eighty nine billion three hundred eighty two million five hundred thirty six thousand one hundred thirty~9789382536130 +Two hundred and fifty four~254 +One hundred forty seven thousand four hundred fifty one~147451 +One million one hundred fifty six thousand one hundred seventy three~1156173 +One billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961 +Ninety seven quadrillion eight hundred eight trillion two hundred sixty four billion seven hundred seventy two million seven hundred ninety two thousand five~97808264772792005 +Seventeen sextillion eight hundred fifty five quintillion thirty six quadrillion six hundred fifty seven trillion seven billion five hundred ninety six million one hundred ten thousand nine hundred forty nine~17855036657007596110949 +Ten quadrillion ten trillion ten million one hundred thousand ten~10010000010100010 +Minus twenty five thousand thirty seven~-25037 +One quadrillion two hundred sixty four trillion three hundred one billion nine hundred thirty eight million one hundred four~1264301938000104 +Minus sixty~-60 +Forty six thousand six hundred sixty four~46664 +Sixty~60 +Zero~Zero +Two million three~2000003 +One thousand thirteen~1013 +One thousand one~1001 +One thousand one hundred~1100 +One thousand twenty six~1026 +One thousand one hundred twenty six~1126 +Eighteen million four hundred fifty thousand nine hundred ninety~18450990 +Eighteen million nine hundred forty thousand seven hundred twenty two~18940722 +Eighteen million six hundred ninety thousand nine hundred sixteen~18690916 +Eighteen thousand eight hundred eighty~18880 +Eleven hundred~1100 +Twenty one hundred~2100 +Twenty one hundred and eleven~2111 +Eleven hundred twenty one~1121 +Nineteen~19 +Twelve~Twelve \ No newline at end of file diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date.txt index f961fbfbb..1147a12c7 100644 --- a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date.txt +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date.txt @@ -13,7 +13,7 @@ two thousand and nine~2009 the twenty fifth of july twenty twelve~25 july 2012 the twenty fifth of july two thousand twelve~25 july 2012 the twenty second of july twenty twelve~22 july 2012 -the fifteenth of january~15 de january +the fifteenth of january~15 january the seventeenth of may twenty ten~17 may 2010 january first~january 1 july twenty second two thousand eight~july 22 2008 diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date_cased.txt new file mode 100644 index 000000000..ab8358dd6 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_date_cased.txt @@ -0,0 +1,20 @@ +Primero De Enero~1 de Enero +Uno de enero~1 de Enero +el uno de Diciembre~el 1 de Diciembre +El primero de diciembre~El 1 de diciembre +Domingo Veintiséis De Octubre~Domingo 26 de Octubre +treinta y uno de diciembre de mil novecientos noventa y dos~31 de diciembre de 1992 +Siglo diecinueve~Siglo xix +doscientos tres antes de Cristo~203 A. C. +January first~January 1 +July twenty second two thousand eight~July 22 2008 +June thirty~June 30 +July twenty fifth twenty twelve~July 25 2012 +Nineteen seventeen~1917 +Twenty twelve~2012 +March sixteen sixty five~March 1665 +Sixteen sixty five~1665 +July two thousand twelve~July 2012 +October nineteen oh five~October 1905 +July fifteen o six~July 1506 +The twenty fifth of july twenty twelve~25 july 2012 \ No newline at end of file diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_decimal_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_decimal_cased.txt new file mode 100644 index 000000000..17b91271d --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_decimal_cased.txt @@ -0,0 +1,19 @@ +Uno coma dos seis~1,26 +Menos uno coma dos seis~-1,26 +Uno Coma Veintiséis~1,26 +Cero coma Dos seis~0,26 +cero coma veintiséis~0,26 +tres coma ciento cuarenta y uno~3,141 +One point two five billion~1.25 billion +Thirteen billion~13 billion +Thirty billion~30 billion +Thirty Billion~30 Billion +Two thousand eight hundred five point eight seven three billion~2805.873 billion +Eighteen~18 +Eighteen point eight five~18.85 +Eighteen point five o~18.50 +Eighteen point five six~18.56 +Eighteen point nine~18.9 +Eighteen point o five~18.05 +Eighteen point one two~18.12 +Eighteen point o one~18.01 diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_electronic_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_electronic_cased.txt new file mode 100644 index 000000000..9131f4f94 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_electronic_cased.txt @@ -0,0 +1,14 @@ +A punto B C Arroba G mail punto com~A.BC@gmail.com +c d f Arroba a b c Punto e d u~cdf@abc.edu +W W W Punto N vidia Punto com~www.nvidia.com +Doble ve doble ve doble ve punto a b c punto es barra e f g~www.abc.es/efg +Doble Ve Doble Ve Doble Ve Punto a b c Punto e s~www.abc.es +N vidia dot com~nvidia.com +Abc at gmail dot com~Abc@gmail.com +Athreed at gmail dot com~Athreed@gmail.com +Kore dot ai~Kore.ai +NVIDIA dot com~NVIDIA.com +NVIDIA dot COM~NVIDIA.COM +WWW.A B C at A B C dot com~WWW.A BC@ABC.com +W W W. A B C dot com~www.ABC.com +w w w . o u r d a i l y n e w s dot com . s m~www.ourdailynews.com . s m \ No newline at end of file diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure.txt index 0a9b90903..01278697f 100644 --- a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure.txt @@ -13,7 +13,7 @@ cuatro segundos~4 s cinco litros~5 l tres metros cúbicos~3 m³ dos kilómetros por hora~2 kph -diez grados farenheit~10 ° f +diez grados farenheit~10 ° F dos metros y medio~2 1/2 m tres quintos de metro~3/5 m menos tres y medio metros por hora~-3 1/2 m/h diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure_cased.txt new file mode 100644 index 000000000..7073a7bf6 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_measure_cased.txt @@ -0,0 +1,22 @@ +Doscientos metros~200 m +tres horas~3 h +una hora~1 h +Doscientos cuarenta y cinco Millas Por Hora~245 mph +Dos Kilos~2 kg +sesenta coma dos cuatro cero cero kilogramos~60,2400 kg +Menos sesenta coma veinticuatro cero cero kilogramos~-60,2400 kg +menos Ocho Coma Cinco Dos por ciento~-8,52 % +uno Porciento~1 % +tres centímetros~3 cm +dos más dos es igual a cuatro~2 + 2 = 4 +Eight point five megawatts~8.5 mW +Eight point five meters~8.5 m +Eight point five two percent~8.52 % +Eight point four four percent~8.44 % +one gigabit per second~1 gbps +nine gigabits per second~9 gbps +five degrees celsius~5 °C +seventy two degrees fahrenheit~72 °F +Seventy two Degrees Fahrenheit~72 °F +two hundred seventy three kelvin~273 K +Nine GigaBits per second~9 gbps \ No newline at end of file diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_money_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_money_cased.txt new file mode 100644 index 000000000..613a822a6 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_money_cased.txt @@ -0,0 +1,16 @@ +doce dólares y cinco centavos~$12,05 +Doce Dólares Y Cinco Céntimos~$12,05 +setenta y cinco Dólares sesenta y tres~$75,63 +Veintinueve dólares cincuenta centavos~$29,50 +Catorce millones quinientos mil Pesos mexicanos~Mex$14500000 +diez pesos Mexicanos~Mex$10 +Two dollars~$2 +One cent~$0.01 +Four united states dollars and sixty nine cents~$4.69 +Seventy five dollars sixty three~$75.63 +Twenty nine dollars fifty cents~$29.50 +Eleven dollars and fifty one cents~$11.51 +Nine hundred ninety three dollars and ninety two cents~$993.92 +Four hundred sixty billion won~₩460 billion +Thirty billion yen~¥30 billion +Two point five billion dollars~$2.5 billion diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_ordinal_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_ordinal_cased.txt new file mode 100644 index 000000000..7b7df1ed7 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_ordinal_cased.txt @@ -0,0 +1,21 @@ +primero~primero +Tercera~Tercera +Primer~Primer +tercer~tercer +Décima~10.ª +undécimo~11.º +Decimoprimer~11.ᵉʳ +Décimo primer~11.ᵉʳ +Décima Primera~11.ª +(technically ungrammatical) décimo primera~(technically ungrammatical) 11.ª +decimotercero~13.º +One hundredth~100th +Twenty five thousand one hundred eleventh~25111th +Second~2nd +Zeroth~0th +First~1st +Second~2nd +Third~3rd +Fourth~4th +Eleventh~11th +Twelfth~12th diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_telephone_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_telephone_cased.txt new file mode 100644 index 000000000..6b9613da0 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_telephone_cased.txt @@ -0,0 +1,19 @@ +Uno dos tres uno dos tres cinco seis siete ocho~123-123-5678 +uno veintitrés uno veintitrés cincuenta y seis setenta y ocho~123-123-5678 +Uno Dos Tres Cuatro Cinco Seis Siete Ocho Nueve~123-456-789 +Triple tres uno dos tres cinco seis siete ocho~333-123-5678 +Más uno uno dos tres uno dos tres cinco seis siete ocho~+1-123-123-5678 +más cincuenta y cuatro uno dos tres uno dos tres cinco seis siete ocho Extensión doce~+54-123-123-5678 ext. 12 +One two three one two three five six seven eight~123-123-5678 +Plus nine one one two three one two three five six seven eight~+91 123-123-5678 +Plus forty four one two three one two three five six seven eight~+44 123-123-5678 +O two three one two three five six seven eight~023-123-5678 +Oh two three one two three five six seven eight~023-123-5678 +Double oh three one two three five six seven eight~003-123-5678 +Two two five dot double five dot o dot four o~225.55.0.40 +Two two five dot double five dot o dot forty five~225.55.0.45 +SSN is seven double nine one two three double one three~SSN is 799-12-3113 +Seven nine nine~799 +A b nine~Ab9 +A b c~A b c +Five w k r a three one~5wkra31 diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time.txt index e12512e3a..17113ad98 100644 --- a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time.txt @@ -21,23 +21,23 @@ las dos de la tarde~las 2:00 p.m. cuarto para las cero~las 23:45 cuarto para las veinticuatro~las 23:45 diez para las doce~las 11:50 -dos y media de la tarde~2:30 p.m. -la una de la tarde u t c más cuatro~la 1:00 p.m. utc+4 -eight oclock g m t~8:00 gmt -seven a m e s t~7:00 a.m. est -two p m~2:00 p.m. -two thirty~2:30 -three o'clock~3:00 -quarter past one~1:15 -half past three~3:30 -eight fifty one~8:51 -eight fifty two~8:52 -eight forty~8:40 -eight nineteen~8:19 -eight o six~8:06 -eight thirty eight~8:38 -eight thirty two~8:32 -eight twenty nine~8:29 +dos y media de la tarde~02:30 p.m. +la una de la tarde u t c más cuatro~la 1:00 p.m. UTC+4 +eight oclock g m t~08:00 gmt +seven a m e s t~07:00 a.m. est +two p m~02:00 p.m. +two thirty~02:30 +three o'clock~03:00 +quarter past one~01:15 +half past three~03:30 +eight fifty one~08:51 +eight fifty two~08:52 +eight forty~08:40 +eight nineteen~08:19 +eight o six~08:06 +eight thirty eight~08:38 +eight thirty two~08:32 +eight twenty nine~08:29 eleven fifty five p m~11:55 p.m. eleven fifty three p m~11:53 p.m. eleven forty a m~11:40 a.m. @@ -47,7 +47,7 @@ eleven forty six a m~11:46 a.m. eleven o six p m~11:06 p.m. eleven thirteen a m~11:13 a.m. half past twelve~12:30 -quarter past one~1:15 +quarter past one~01:15 quarter to one~12:45 quarter to twelve~11:45 set alarm at ten to eleven pm~set alarm at 10:50 p.m. diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time_cased.txt new file mode 100644 index 000000000..875ff97d6 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_time_cased.txt @@ -0,0 +1,19 @@ +las dieciséis cincuenta~las 16:50 +la una~la una +Las dos~Las dos +Las tres personas~Las tres personas +Las Dos a eme~Las 2:00 a.m. +la una Pe Eme~la 1:00 P.M. +la una y diez~la 1:10 +la una y Diez a eme~la 1:10 a.m. +La Una Y Diez pe eme~La 1:10 p.m. +Eight oclock g m t~08:00 gmt +Seven a m e s t~07:00 a.m. est +Two p m~02:00 p.m. +Seven A M E S T~07:00 A.M. EST +Two P M~02:00 P.M. +Two thirty~02:30 +Set alarm at ten to eleven pm~Set alarm at 10:50 p.m. +One min to one am~12:59 a.m. +eleven Forty six A M~11:46 A.M. +eleven forty six AM~11:46 A.M. \ No newline at end of file diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_whitelist.txt index 895fb52c2..90c024aa1 100644 --- a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_whitelist.txt @@ -1,8 +1,8 @@ -usted~ud. -ustedes~uds. -habla usted español~habla ud. español -hablan ustedes español~hablan uds. español -estados unidos~ee. uu. +usted~Ud. +ustedes~Uds. +habla usted español~habla Ud. español +hablan ustedes español~hablan Uds. español +estados unidos~EE. UU. doctor dao~dr. dao misses smith~mrs. smith mister dao~mr. dao diff --git a/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_word_cased.txt b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_word_cased.txt new file mode 100644 index 000000000..ebe0c0864 --- /dev/null +++ b/tests/nemo_text_processing/es_en/data_inverse_text_normalization/test_cases_word_cased.txt @@ -0,0 +1,17 @@ +~ +Yahoo!~Yahoo! +Veinte!~20 ! +X ~X +—~— +AAA~AAA +Aabach~Aabach +aabenraa~aabenraa +Aachen's~Aachen's +Aadri~Aadri +Aaliyan's~Aaliyan's +Aahar~Aahar +Aahh~Aahh +Aahperd~Aahperd +Aaibinterstate~Aaibinterstate +Aajab~Aajab +Aakasa~Aakasa \ No newline at end of file diff --git a/tests/nemo_text_processing/es_en/test_cardinal.py b/tests/nemo_text_processing/es_en/test_cardinal.py index 02c623fb6..29fed2b30 100644 --- a/tests/nemo_text_processing/es_en/test_cardinal.py +++ b/tests/nemo_text_processing/es_en/test_cardinal.py @@ -13,15 +13,18 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestCardinal: - inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_cardinal.txt')) @pytest.mark.run_only_on('CPU') @@ -29,3 +32,10 @@ class TestCardinal: def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_cardinal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_date.py b/tests/nemo_text_processing/es_en/test_date.py index 0cac5dabd..8368542d0 100644 --- a/tests/nemo_text_processing/es_en/test_date.py +++ b/tests/nemo_text_processing/es_en/test_date.py @@ -13,14 +13,18 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestDate: inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_date.txt')) @pytest.mark.run_only_on('CPU') @@ -28,3 +32,10 @@ class TestDate: def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_date_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_decimal.py b/tests/nemo_text_processing/es_en/test_decimal.py index 01c9fcf4c..22a0e4d2c 100644 --- a/tests/nemo_text_processing/es_en/test_decimal.py +++ b/tests/nemo_text_processing/es_en/test_decimal.py @@ -13,14 +13,18 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestDecimal: inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_decimal.txt')) @pytest.mark.run_only_on('CPU') @@ -28,3 +32,10 @@ class TestDecimal: def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_decimal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_electronic.py b/tests/nemo_text_processing/es_en/test_electronic.py index d3c4a921b..90ab9e0d2 100644 --- a/tests/nemo_text_processing/es_en/test_electronic.py +++ b/tests/nemo_text_processing/es_en/test_electronic.py @@ -13,18 +13,29 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestElectronic: - inverse_normalizer_es = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_electronic.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm_es(self, test_input, expected): - pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_electronic_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_fraction.py b/tests/nemo_text_processing/es_en/test_fraction.py index c4866bf7a..125d9fa3d 100644 --- a/tests/nemo_text_processing/es_en/test_fraction.py +++ b/tests/nemo_text_processing/es_en/test_fraction.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/es_en/test_measure.py b/tests/nemo_text_processing/es_en/test_measure.py index 56defb8a8..94fc46f1d 100644 --- a/tests/nemo_text_processing/es_en/test_measure.py +++ b/tests/nemo_text_processing/es_en/test_measure.py @@ -14,18 +14,29 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestMeasure: - inverse_normalizer_es = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_measure.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm_es(self, test_input, expected): - pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_measure_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_money.py b/tests/nemo_text_processing/es_en/test_money.py index 3d3e75656..8dadbe3d1 100644 --- a/tests/nemo_text_processing/es_en/test_money.py +++ b/tests/nemo_text_processing/es_en/test_money.py @@ -14,18 +14,29 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestMoney: inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_money.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_denorm(self, test_input, expected): + def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_money_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_ordinal.py b/tests/nemo_text_processing/es_en/test_ordinal.py index 0b4a9cad7..bfb3dfc84 100644 --- a/tests/nemo_text_processing/es_en/test_ordinal.py +++ b/tests/nemo_text_processing/es_en/test_ordinal.py @@ -14,18 +14,29 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestOrdinal: inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_ordinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_denorm(self, test_input, expected): + def test_denorm_es(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_ordinal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/es_en/test_sparrowhawk_inverse_text_normalization.sh index 98554f619..6765d360c 100644 --- a/tests/nemo_text_processing/es_en/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/es_en/test_sparrowhawk_inverse_text_normalization.sh @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/es_en/test_telephone.py b/tests/nemo_text_processing/es_en/test_telephone.py index 4b86eeb94..112bf0949 100644 --- a/tests/nemo_text_processing/es_en/test_telephone.py +++ b/tests/nemo_text_processing/es_en/test_telephone.py @@ -14,18 +14,29 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestTelephone: - inverse_normalizer_es = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_telephone.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm_es(self, test_input, expected): - pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_telephone_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_time.py b/tests/nemo_text_processing/es_en/test_time.py index 1fbba3d90..f1fc03945 100644 --- a/tests/nemo_text_processing/es_en/test_time.py +++ b/tests/nemo_text_processing/es_en/test_time.py @@ -13,18 +13,29 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestTime: - inverse_normalizer_es = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_time.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm_es(self, test_input, expected): - pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_time_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) assert pred == expected diff --git a/tests/nemo_text_processing/es_en/test_whitelist.py b/tests/nemo_text_processing/es_en/test_whitelist.py index e42b9b179..13924220b 100644 --- a/tests/nemo_text_processing/es_en/test_whitelist.py +++ b/tests/nemo_text_processing/es_en/test_whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/es_en/test_word.py b/tests/nemo_text_processing/es_en/test_word.py index 3cd465165..b7b67a17b 100644 --- a/tests/nemo_text_processing/es_en/test_word.py +++ b/tests/nemo_text_processing/es_en/test_word.py @@ -13,18 +13,29 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file class TestWord: - inverse_normalizer_es = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer = InverseNormalizer(lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='es_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) @parameterized.expand(parse_test_case_file('es_en/data_inverse_text_normalization/test_cases_word.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm_es(self, test_input, expected): - pred = self.inverse_normalizer_es.inverse_normalize(test_input, verbose=False) + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_word_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) assert pred == expected diff --git a/tests/nemo_text_processing/fr/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/fr/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..3b4f09154 --- /dev/null +++ b/tests/nemo_text_processing/fr/data_text_normalization/test_cases_date.txt @@ -0,0 +1,13 @@ +02.03.2003~deux mars deux mille trois +02/03/2003~deux mars deux mille trois +02-03-2003~deux mars deux mille trois +le 02.03.2003~le deux mars deux mille trois +17.06~dix-sept juin +17 janvier~dix-sept janvier +10 mars 2023~dix mars deux mille vingt-trois +le 10 mars 2023~le dix mars deux mille vingt-trois +les 80s~les eighties +les 17/18 juin~les dix-sept dix-huit juin +les 17/18/19 mars~les dix-sept dix-huit dix-neuf mars +les 17-18-19 juin~les dix-sept dix-huit dix-neuf juin +les 17-18-19 juin 2025~les dix-sept dix-huit dix-neuf juin deux mille vingt-cinq \ No newline at end of file diff --git a/tests/nemo_text_processing/fr/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/fr/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..6a6b179dc --- /dev/null +++ b/tests/nemo_text_processing/fr/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,7 @@ +Dʳ~docteur +Dʳᵉˢ~docteures +Mᵐᵉ~madame +Mᵐᵉˢ~mesdames +Mˡˡᵉ~mademoiselle +Mˡˡᵉˢ~mademoiselles +18%~dix-huit pour cent \ No newline at end of file diff --git a/tests/nemo_text_processing/fr/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/fr/data_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..81d8cfd0a --- /dev/null +++ b/tests/nemo_text_processing/fr/data_text_normalization/test_cases_word.txt @@ -0,0 +1,49 @@ +~ +yahoo!~yahoo! +20 !~vingt ! +x~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang \ No newline at end of file diff --git a/tests/nemo_text_processing/fr/test_cardinal.py b/tests/nemo_text_processing/fr/test_cardinal.py index 0899ccef4..4e3353cef 100644 --- a/tests/nemo_text_processing/fr/test_cardinal.py +++ b/tests/nemo_text_processing/fr/test_cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_date.py b/tests/nemo_text_processing/fr/test_date.py index 754870153..35e3086cd 100644 --- a/tests/nemo_text_processing/fr/test_date.py +++ b/tests/nemo_text_processing/fr/test_date.py @@ -13,9 +13,11 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -28,3 +30,12 @@ class TestDate: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(input_case='cased', lang='fr', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('fr/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/fr/test_decimal.py b/tests/nemo_text_processing/fr/test_decimal.py index d5891074c..54e8f53c1 100644 --- a/tests/nemo_text_processing/fr/test_decimal.py +++ b/tests/nemo_text_processing/fr/test_decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_electronic.py b/tests/nemo_text_processing/fr/test_electronic.py index 2fb86e157..475af6ffd 100644 --- a/tests/nemo_text_processing/fr/test_electronic.py +++ b/tests/nemo_text_processing/fr/test_electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_fraction.py b/tests/nemo_text_processing/fr/test_fraction.py index a669d58ca..5b5431b79 100644 --- a/tests/nemo_text_processing/fr/test_fraction.py +++ b/tests/nemo_text_processing/fr/test_fraction.py @@ -14,9 +14,10 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_measure.py b/tests/nemo_text_processing/fr/test_measure.py index f48fb663e..884ccc957 100644 --- a/tests/nemo_text_processing/fr/test_measure.py +++ b/tests/nemo_text_processing/fr/test_measure.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_money.py b/tests/nemo_text_processing/fr/test_money.py index 818d7b4f7..bba19432e 100644 --- a/tests/nemo_text_processing/fr/test_money.py +++ b/tests/nemo_text_processing/fr/test_money.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_ordinal.py b/tests/nemo_text_processing/fr/test_ordinal.py index 584df3440..a5915dcb5 100644 --- a/tests/nemo_text_processing/fr/test_ordinal.py +++ b/tests/nemo_text_processing/fr/test_ordinal.py @@ -14,9 +14,10 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/fr/test_sparrowhawk_inverse_text_normalization.sh index 4ca12af7f..0a8336f68 100644 --- a/tests/nemo_text_processing/fr/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/fr/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,16 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars_itn_grammars_lower_cased # read test file while read testcase; do diff --git a/tests/nemo_text_processing/fr/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/fr/test_sparrowhawk_normalization.sh index c845f44cd..71f3f4759 100644 --- a/tests/nemo_text_processing/fr/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/fr/test_sparrowhawk_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do @@ -25,6 +27,11 @@ testTNCardinal() { runtest $input } +testTNDate() { + input=$PROJECT_DIR/fr/data_text_normalization/test_cases_date.txt + runtest $input +} + testTNDecimal() { input=$PROJECT_DIR/fr/data_text_normalization/test_cases_decimal.txt runtest $input @@ -40,5 +47,15 @@ testTNOrdinal() { runtest $input } +testTNWhitelist() { + input=$PROJECT_DIR/fr/data_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testTNWord() { + input=$PROJECT_DIR/fr/data_text_normalization/test_cases_word.txt + runtest $input +} + # Load shUnit2 . $PROJECT_DIR/../shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/fr/test_telephone.py b/tests/nemo_text_processing/fr/test_telephone.py index 8b062f3c8..dd24dccfe 100644 --- a/tests/nemo_text_processing/fr/test_telephone.py +++ b/tests/nemo_text_processing/fr/test_telephone.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_time.py b/tests/nemo_text_processing/fr/test_time.py index 6dec74c52..f5eee9db2 100644 --- a/tests/nemo_text_processing/fr/test_time.py +++ b/tests/nemo_text_processing/fr/test_time.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/fr/test_whitelist.py b/tests/nemo_text_processing/fr/test_whitelist.py index 075584b3b..dac398fba 100644 --- a/tests/nemo_text_processing/fr/test_whitelist.py +++ b/tests/nemo_text_processing/fr/test_whitelist.py @@ -14,9 +14,11 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -29,3 +31,12 @@ class TestWhitelist: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='fr', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('fr/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/fr/test_word.py b/tests/nemo_text_processing/fr/test_word.py index 46057117f..6d48db4fc 100644 --- a/tests/nemo_text_processing/fr/test_word.py +++ b/tests/nemo_text_processing/fr/test_word.py @@ -14,9 +14,11 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -29,3 +31,12 @@ class TestWord: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='fr', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('fr/data_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/he/__init__.py b/tests/nemo_text_processing/he/__init__.py new file mode 100644 index 000000000..bc443be41 --- /dev/null +++ b/tests/nemo_text_processing/he/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..cfb6f8db0 --- /dev/null +++ b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,138 @@ +אפס~אפס +מינוס שלוש~-3 +עשר~עשר +שלוש עשרה~13 +שלושה עשר~13 +עשרים~20 +עשרים ותשע~29 +עשרים ותשעה~29 +ארבעים~40 +מינוס ארבעים ושש~-46 +שבעים ושבעה~77 +מאה~100 +מאה ואחת~101 +מאה ועשר~110 +מאה ושש עשרה~116 +מאה עשרים~120 +מאה ועשרים~120 +כמאה עשרים וחמש~כ-125 +מאתיים~200 +מאתיים ושלוש~203 +מאתיים שלושים~230 +שלוש מאות ושלושים~330 +מינוס מאתיים שישים ושבע~-267 +ארבע מאות~400 +כחמש מאות עובדים~כ-500 עובדים +חמש מאות שבעים ותשע~579 +תשע מאות תשעים~990 +תשע מאות תשעים ותשע~999 +אלף~1,000 +אלף וארבע~1,004 +אלף עשרים ושמונה~1,028 +אלף מאה וחמש~1,105 +אלף מאה שלושים~1,130 +אלף תשע מאות תשעים ואחת~1,991 +אלפיים~2,000 +אלפיים וחמש~2,005 +אלפיים ועשר~2,010 +אלפיים ואחת עשרה~2,011 +אלפיים מאה~2,100 +אלפיים מאתיים~2,200 +מינוס אלפיים מאתיים עשרים ושתיים~-2,222 +אלפיים שלוש מאות~2,300 +אלפיים ארבע מאות ושבע~2,407 +מינוס אלפיים ארבע מאות שבעים~-2,470 +מינוס אלפיים ארבע מאות שבעים וחמש~-2,475 +שלושת אלפים~3,000 +שלושת אלפים וחמש~3,005 +שלושת אלפים ועשר~3,010 +שלושת אלפים וארבע עשרה~3,014 +שלושת אלפים מאה~3,100 +שלושת אלפים מאתיים~3,200 +מינוס שלושת אלפים שבע מאות עשרים ואחת~-3,721 +שלושת אלפים שמונה מאות~3,800 +שלושת אלפים ושמונה מאות~3,800 +שלושת אלפים תשע מאות ושבע~3,907 +מינוס שלושת אלפים מאתיים ועשרים~-3,220 +חמשת אלפים~5,000 +תשעת אלפים תשע מאות תשעים ותשע~9,999 +עשרת אלפים~10,000 +עשרת אלפים ואחת~10,001 +עשרת אלפים וחמש עשרה~10,015 +עשרת אלפים ועשרים~10,020 +עשרת אלפים עשרים ושלוש~10,023 +עשרת אלפים מאתיים~10,200 +עשרת אלפים מאתיים ואחד~10,201 +עשרת אלפים מאתיים ארבעים~10,240 +עשרת אלפים מאתיים וארבעים~10,240 +עשרת אלפים שלוש מאות חמישים~10,350 +עשרת אלפים שלוש מאות וחמישים~10,350 +שתיים עשרה אלף שש מאות~12,600 +שתיים עשרה אלף ושש מאות~12,600 +שתיים עשרה אלף שש מאות ואחת~12,601 +כשמונים ושבע אלף ועשר~כ-87,010 +תשעים ותשע אלף תשע מאות תשעים ותשע~99,999 +מאה אלף~100,000 +כמאה אלף תושבים~כ-100,000 תושבים +מאה אלף ושלוש~100,003 +מאה אלף ושתיים עשרה~100,012 +מאה אלף וארבעים~100,040 +מאה אלף ארבעים ושבע~100,047 +מאה אלף וארבעים ושבע~100,047 +מאה אלף ומאה~100,100 +מאה אלף מאה~100,100 +מאה אלף מאה שלושים ושלוש~100,133 +מאה ואחד אלף~101,000 +מאה ואחד אלף ואחת~101,001 +מאה ואחד אלף ועשר~101,010 +מאה ואחד אלף ואחת עשרה~101,011 +מאה ואחד אלף מאתיים~101,200 +כמאה ואחד אלף ומאתיים~כ-101,200 +מינוס מאה ואחת אלף מאתיים ועשרים~-101,220 +מינוס מאה ואחת אלף מאתיים עשרים~-101,220 +מינוס מאה ואחת אלף מאתיים עשרים ותשע~-101,229 +מינוס מאה ואחת אלף מאתיים עשרים ותשע~-101,229 +מאה ושתיים אלף~102,000 +מאה ושלוש אלף חמש מאות~103,500 +מאה ושלוש אלף וחמש מאות~103,500 +מאה וארבע אלף חמש מאות וארבע~104,504 +מאתיים ארבעים אלף~240,000 +מאתיים וארבעים אלף~240,000 +מאתיים חמישים וחמש אלף ושש~255,006 +מאתיים חמישים וחמש אלף וארבע מאות ושש~255,406 +מאתיים חמישים וחמש אלף ארבע מאות ושש~255,406 +חמש מאות חמישים וחמש אלף~555,000 +תשע מאות תשעים ותשע אלף תשע מאות תשעים ותשע~999,999 +מיליון~1,000,000 +מיליון ואחת~1,000,001 +מיליון ועשר~1,000,010 +מיליון חמש עשרה~1,000,015 +מיליון ושבעים~1,000,070 +מיליון שבעים~1,000,070 +מיליון ארבע מאות~1,000,400 +מיליון וארבע מאות~1,000,400 +מיליון ארבע מאות עשרים~1,000,420 +מיליון ארבע מאות ועשרים~1,000,420 +מיליון אלף~1,001,000 +מיליון שלושת אלפים~1,003,000 +מיליון ואלף~1,001,000 +מיליון אלף ואחת~1,001,001 +שלושה מיליון אלף~3,001,000 +שלושה מיליוןאלף וחמש~3,001,005 +שלושה מיליון ארבעים ושלוש אלף~3,043,000 +שלושה מיליון ארבעים ושלוש אלף ואחת~3,043,001 +שלושה מיליון ארבעים ושלוש אלף ושישים ואחת~3,043,061 +שלושה מיליון ארבעים ושלוש אלף שישים ואחת~3,043,061 +שלושה מיליון חמש מאות ארבעים ושלוש אלף~3,543,000 +שלושה מיליון חמש מאות ארבעים ושלוש אלף ושבע~3,543,007 +מינוס שלושה מיליון חמש מאות ארבעים ושלוש אלף ושבע~-3,543,007 +עשר מיליון~10 מיליון +עשרה מיליון~10 מיליון +עשרים מיליון~20 מיליון +חמש עשרה מיליון~15 מיליון +שלוש עשרה מיליון ארבעים ושלוש אלף~13,043,000 +מאה מיליון~100 מיליון +מאה עשרים ושתיים מיליון~122 מיליון +מאה עשרים ושתיים מיליון ושלוש עשרה~122,000,013 +מאה עשרים ושתיים מיליון חמישים אלף ושלוש עשרה~122,050,013 +שלוש אלף~3,000 diff --git a/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..96b745de4 --- /dev/null +++ b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,29 @@ +אחד במאי אלף תשע מאות שמונים ושלוש~1.5.1983 +השתיים עשרה לשתיים עשרה אלף תשע מאות תשעים ואחת~ה-12.12.1991 +השתיים עשרה בדצמבר אלף תשע מאות תשעים ואחת~ה-12.12.1991 +בינואר עשרים עשרים ואחת~בינואר 2021 +בשלישי לשלישי אלף תשע מאות תשעים~ב-3.3.1990 +העשירי באפריל~ה-10.4 +אחד במאי~1.5 +הראשון לחמישי~הראשון לחמישי +יוני אלפיים וחמש עשרה~יוני 2015 +ביוני אלפיים וחמש עשרה~ביוני 2015 +מתחיל בספטמבר עשרים עשרים~מתחיל בספטמבר 2020 +בשבעה עשר באוגוסט~ב-17.8 +בשבעה עשר באוגוסט עשרים שלושים~ב-17.8.2030 +בשבעה עשר לשמיני עשרים שלושים~ב-17.8.2030 +עשרים ושישי לרביעי עשרים עשרים וארבע~26.4.2024 +עשרים ושש לרביעי עשרים עשרים וארבע~26.4.2024 +עשרים ושישי לאפריל עשרים עשרים וארבע~26.4.2024 +עשרים ושש באפריל עשרים עשרים וארבע~26.4.2024 +עשרים ושישי לרביעי עשרים וארבע~26.4.24 +עשרים ושש לרביעי עשרים וארבע~26.4.24 +עשרים ושישי לאפריל עשרים וארבע~26.4.24 +עשרים ושש באפריל עשרים וארבע~26.4.24 +עשרים ושישה באפריל עשרים וארבע~26.4.24 +אנשים לא ידעו אחד מהשני~אנשים לא ידעו אחד מהשני +בשבעה באוקטובר~ב-7.10 +בשנת אלפיים וחמש~בשנת 2005 +משנת עשרים עשרים ואחת~משנת 2021 +השנה אלפיים ושלוש~השנה 2003 +שנת אלפיים וארבע~שנת 2004 \ No newline at end of file diff --git a/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..b864e264a --- /dev/null +++ b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,66 @@ +חמש נקודה שתיים מיליון~5.2 מיליון +מאה שישים וארבע נקודה חמישים ושמונה אלף~164.58 אלף +ארבע מאות מיליון~400 מיליון +חמישים מיליארד~50 מיליארד +ארבע מאות וחמש מיליארד~405 מיליארד +ארבע נקודה שמונים וחמש מיליארד~4.85 מיליארד +מאה מיליארד~100 מיליארד +מאה ועשר מיליארד~110 מיליארד +מאה שלושים ושתיים מיליארד~132 מיליארד +אחד נקודה שמונים וארבע מיליארד~1.84 מיליארד +אחד נקודה שמונים ואחת מיליארד~1.81 מיליארד +אחד נקודה חמש תשע מיליארד~1.59 מיליארד +אחד נקודה ארבע חמש שלוש מיליארד~1.453 מיליארד +אחד נקודה שבעים ושתיים מיליארד~1.72 מיליארד +אחד נקודה שתיים חמש מיליארד~1.25 מיליארד +שלוש עשרה מיליארד~13 מיליארד +שלושים מיליארד~30 מיליארד +אלפיים שמונה מאות וחמש נקודה שמונה שבע שלוש מיליון~2,805.873 מיליון +עשרה מיליון~10 מיליון +עשר מיליון~10 מיליון +חמש מיליון~5 מיליון +חמש מאות מיליון~500 מיליון +שתיים עשרה מיליון~12 מיליון +שניים עשר מיליון~12 מיליון +שלוש עשרה מיליון~13 מיליון +ארבע מיליון~4 מיליון +ארבעים וחמש מיליון~45 מיליון +חמש עשרה מיליארד~15 מיליארד +שני מיליון~2 מיליון +שתי מיליון~2 מיליון +שמונה מיליון~8 מיליון +מינוס שישים נקודה שתיים ארבע אפס אפס~-60.2400 +אפס נקודה עשרים ושש~0.26 +אפס נקודה שתיים שש~0.26 +שישים נקודה שתיים~60.2 +שמונה עשרה נקודה שמונים וחמש~18.85 +שמונה עשרה נקודה חמש אפס~18.50 +שמונה עשרה נקודה חמישים ושש~18.56 +שמונה עשרה נקודה תשע~18.9 +שמונה עשרה נקודה אפס חמש~18.05 +שמונה עשרה נקודה שתיים עשרה~18.12 +שמונה עשרה נקודה אפס אחד~18.01 +שמונה עשרה נקודה אפס אפס אפס~18.000 +שמונה עשרה נקודה שש~18.6 +שמונה עשרה נקודה שלוש אפס אפס~18.300 +שמונה עשרה נקודה שלושים ושש~18.36 +שמונה עשרה נקודה שתיים חמש~18.25 +שמונה עשרה נקודה עשרים ושתיים~18.22 +שמונה מאות ושמונה עשרה נקודה שלוש אפס שלוש~818.303 +שמונה מאות ושמונה נקודה שמונה~808.8 +שמונה מאות ושמונה נקודה אפס~808.0 +שמונה מאות שמונים ושמונה נקודה אחד~888.1 +שמונה מאות שמונים וארבע נקודה שלוש~884.3 +שמונה מאות שמונים ושתיים נקודה שמונה~882.8 +שמונה מאות שמונים ושתיים נקודה אפס~882.0 +שמונה מאות ארבעים וחמש נקודה תשעים וארבע~845.94 +שבעים ותשע וחצי~79.5 +שתיים ורבע~שתיים ורבע +שלוש ועשירית~3.1 +מינוס שלוש וחצי~-3.5 +עשר ושתי עשיריות~10.2 +שתיים ושלושת רבעי~2.75 +שתיים עשרה אלף ושתיים עשרה נקודה שתיים עשרה~12,012.12 +שתים עשרה אלף ושתים עשרה נקודה שתים עשרה~12,012.12 +שתיים ועשירית~2.1 +אחת ועשירית~1.1 diff --git a/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..3d0a40a07 --- /dev/null +++ b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,10 @@ +מינוס חמש עשרה אחוז~-15% +חמש עשרה אחוז~15% +מינוס שתים עשרה נקודה חמש מעלות ~-12.5° +שתיים עשרה נקודה חמש מעלות~12.5° +שתיים עשרה נקודה חמש מעלות צלסיוס~12.5°C +אלף אחוזים~1,000% +אחוז אחד~1% +מאתיים חמישים גרם~250 ג׳ +סנטימטר אחד~1 ס״מ +שלוש מיליגרם~3 מ״ג diff --git a/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..0f6464445 --- /dev/null +++ b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,34 @@ +בשעה חמש בצהריים~בשעה 17:00 בצהריים +בחמש בצהריים~ב-17:00 בצהריים +רבע לשש בבוקר~5:45 בבוקר +בתשע בבוקר~ב-9:00 בבוקר +השעה עשרים וחמישה לאחת בצהריים~השעה 12:35 בצהריים +נפגשנו באחת ושתי דקות בצהריים~נפגשנו ב-13:02 בצהריים +נפגשנו באחת ושלוש דקות בצהריים~נפגשנו ב-13:03 בצהריים +נפגשנו באחת וחמישה בצהריים~נפגשנו ב-13:05 בצהריים +שתיים ועשרה בבוקר~2:10 בבוקר +בשעה שתיים ועשרה בצהריים~בשעה 14:10 בצהריים +בשתיים ועשרה אחרי הצהריים~ב-14:10 אחרי הצהריים +שלוש ודקה בצהריים~15:01 בצהריים +ארבע ושלוש דקות אחרי הצהריים~16:03 אחרי הצהריים +שש ועשרים דקות בערב~18:20 בערב +בשש וחצי בערב~ב-18:30 בערב +חמישה לשלוש בבוקר~2:55 בבוקר +רבע לשש בערב~17:45 בערב +שלוש בצהריים~15:00 בצהריים +אחת לפנות בוקר~1:00 לפנות בוקר +אתמול בחמש אחרי הצהריים יצאנו עם אמא למכולת ובדרך ראינו שהגן שלנו סגור~אתמול ב-17:00 אחרי הצהריים יצאנו עם אמא למכולת ובדרך ראינו שהגן שלנו סגור +חמישה לחצות~23:55 +ברבע לחצות~ב-23:45 +בשעה חצות ועשרה~בשעה 0:10 +בחצות ודקה~ב-0:01 +חצות ושתיים עשרה דקות~0:12 +שלוש דקות לחצות~23:57 +חצות ושתי דקות~0:02 +חצות~0:00 +דקה לשלוש בצהריים~14:59 בצהריים +הפגישה זזה משבע בבוקר לשמונה וחצי בבוקר~הפגישה זזה מ-7:00 בבוקר ל-8:30 בבוקר +באחת בלילה~ב-1:00 בלילה +חמש לפנות ערב~17:00 לפנות ערב +בשלוש לפנות בוקר~ב-3:00 לפנות בוקר +עשרים לחמש אחרי הצהריים~16:40 אחרי הצהריים \ No newline at end of file diff --git a/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..67e4d6560 --- /dev/null +++ b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,4 @@ +בשנת שבעים לפני הספירה~בשנת 70 לפנה״ס +יש מאתיים חמישים עורכי דין חדשים~יש 250 עו״ד חדשים +ישראל היא המדינה החמישים ואחת של ארצות הברית~ישראל היא המדינה ה-51 של ארה״ב +דוקטור שמילוביץ רשם לי תרופה חדשה~ד״ר שמילוביץ רשם לי תרופה חדשה \ No newline at end of file diff --git a/tests/nemo_text_processing/he/data_inverse_text_normalization/test_full_sentences.txt b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_full_sentences.txt new file mode 100644 index 000000000..1bf28b0fb --- /dev/null +++ b/tests/nemo_text_processing/he/data_inverse_text_normalization/test_full_sentences.txt @@ -0,0 +1,56 @@ +אתמול בשעה שבע וחצי בבוקר היה לי תור לרופא~אתמול בשעה 7:30 בבוקר היה לי תור לרופא +הגעתי למרפאה בשבע ורבע בבוקר כדי לא לאחר~הגעתי למרפאה ב-7:15 בבוקר כדי לא לאחר +אמרתי לרופא שאני בן חמישים ושלוש שאני נשוי ויש לי שלושה ילדים.~אמרתי לרופא שאני בן 53 שאני נשוי ויש לי שלושה ילדים. +אמרתי לו שיש לי כאבים ביד, אז הוא בדק אותי ~אמרתי לו שיש לי כאבים ביד, אז הוא בדק אותי +הוא אמר שיש חשד לשבר באמה של בין שני סנטימטר לארבע סנטימטר ושנצטרך לעשות צילום כדי לדעת~הוא אמר שיש חשד לשבר באמה של בין 2 ס״מ ל-4 ס״מ ושנצטרך לעשות צילום כדי לדעת +בינתיים הוא רשם לי עשר מיליגרם של משככי כאבים~בינתיים הוא רשם לי 10 מ״ג של משככי כאבים +הוא אמר שזה מאוד נפוץ ושליותר מעשר אחוז מהאוכלוסיה יש את זה~הוא אמר שזה מאוד נפוץ ושליותר מ-10% מהאוכלוסיה יש את זה +הוא העריך את סיכויי ההחלמה בשמונים ושלוש נקודה שש אחוז~הוא העריך את סיכויי ההחלמה ב-83.6% +בסוף המפגש הוא קבע ביקורת לשמיני באוגוסט~בסוף המפגש הוא קבע ביקורת ל-8.8 +יש לי חמישה תפוחים~יש לי חמישה תפוחים +אף אחד לא רוצה~אף אחד לא רוצה +בכל כיתה יש עשרים, עשרים ושתיים תלמידים~בכל כיתה יש 20 , 22 תלמידים +בכל כיתה יש עשרים - עשרים ושתיים תלמידים~בכל כיתה יש 20 - 22 תלמידים +אחת עשרה אלף שבע מאות חמישים ושש~11,756 +ע"פ הנתונים החדשים שקיבלנו הייתה עלייה של שלושים נקודה שתיים עשרה אחוז במכירות~ע"פ הנתונים החדשים שקיבלנו הייתה עלייה של 30.12% במכירות +אני בטוח בזה במאה אחוז~אני בטוח בזה ב-100% +יש לזה שלושים ותשע נקודה שישים ושבע אחוז הצלחה~יש לזה 39.67% הצלחה +יהי אפסילון אפס ויהי איקס~יהי אפסילון אפס ויהי איקס +לשתינו יש שתי בנות~לשתינו יש שתי בנות +היום יום שני ומחר יום שלישי~היום יום שני ומחר יום שלישי +שלוש וחצי קילוגרם~3.5 ק״ג +חמש ורבע סנטימטר~5.25 ס״מ +שמונה ושלושת רבעי~8.75 +שמונה ורבע מיליון~8.25 מיליון +שתיים וחצי~שתיים וחצי +שתיים וחצי מיליון~2.5 מיליון +בשתיים וחצי מיליון~ב-2.5 מיליון +שתיים וחצי בבוקר~2:30 בבוקר +מינוס שלוש וחצי אחוז~-3.5% +שלוש וחצי~שלוש וחצי +עשרת אלפים ומאתיים ארבעים~10,240 +אפס מאופס~אפס מאופס +הוא מתנהג כמו אפס.... בקיצור כל עניין האפס~הוא מתנהג כמו אפס.... בקיצור כל עניין האפס +מאה שישים וארבע נקודה חמישים ושמונה אלף~164.58 אלף +הפגישה זזה משבע וחצי בבוקר לשמונה~הפגישה זזה מ-7:30 בבוקר לשמונה +על סמך זה יצאנו ביום ראשון~על סמך זה יצאנו ביום ראשון +צעירים היו בגיל שלושים שלושים וחמש~צעירים היו בגיל 30 35 +אולי שניים שלושה~אולי שניים שלושה +בן הראשון שלי שנולד~בן הראשון שלי שנולד +אנחנו היינו איזה חמישה עשר איש~אנחנו היינו איזה 15 איש +התחילו לחזור וחזרו אחד אחד~התחילו לחזור וחזרו אחד אחד +וזה היה כבר אולי שעה תשע~וזה היה כבר אולי שעה תשע +אני מדבר על שמונה עשר באפריל~אני מדבר על 18.4 +שמונה עשר בינואר~18.1 +הייתה נראית כעת שתיים עשרה שלוש עשרה~הייתה נראית כעת 12 13 +היה בערך בעשירי בעשירי למאי~היה בערך בעשירי ב-10.5 +באמצע הלילה שתיים בלילה~באמצע הלילה 2:00 בלילה +למחרת בשעה חמש~למחרת בשעה חמש +בשנת אלף תשע מאות ארבעים ושמונה~בשנת 1948 +באלף תשע מאות ארבעים ושמונה~ב-1,948 +ארבע מאות וחמישים מיליגרם~450 מ״ג +ארבע וחצי~ארבע וחצי +יהי אפסילון אפס ויהי איקס~יהי אפסילון אפס ויהי איקס +לשתינו יש שתי בנות~לשתינו יש שתי בנות +מחר בשש וחצי בבוקר נעלה על האוטובוסים ונצא לטיול השנתי. בשעה שמונה נגיע למצדה ונתחיל לטפס למעלה, נהיה שם עד אחת וחצי בצהריים, אולי רבע לשתיים, ונרד בשביל הנחש~מחר ב-6:30 בבוקר נעלה על האוטובוסים ונצא לטיול השנתי. בשעה שמונה נגיע למצדה ונתחיל לטפס למעלה, נהיה שם עד 13:30 בצהריים , אולי 1:45 , ונרד בשביל הנחש +יש לי חמישה תפוחים~יש לי חמישה תפוחים \ No newline at end of file diff --git a/tests/nemo_text_processing/he/test_cardinal.py b/tests/nemo_text_processing/he/test_cardinal.py new file mode 100644 index 000000000..4700725b1 --- /dev/null +++ b/tests/nemo_text_processing/he/test_cardinal.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + inverse_normalizer_he = InverseNormalizer(lang='he', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('he/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_he.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/he/test_date.py b/tests/nemo_text_processing/he/test_date.py new file mode 100644 index 000000000..73c183e7b --- /dev/null +++ b/tests/nemo_text_processing/he/test_date.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + inverse_normalizer_he = InverseNormalizer(lang='he', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('he/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_he.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/he/test_decimal.py b/tests/nemo_text_processing/he/test_decimal.py new file mode 100644 index 000000000..125fc31d0 --- /dev/null +++ b/tests/nemo_text_processing/he/test_decimal.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + inverse_normalizer_he = InverseNormalizer(lang='he', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('he/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_he.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/he/test_full_sentences.py b/tests/nemo_text_processing/he/test_full_sentences.py new file mode 100644 index 000000000..0bc9251a7 --- /dev/null +++ b/tests/nemo_text_processing/he/test_full_sentences.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFullSentences: + inverse_normalizer_he = InverseNormalizer(lang='he', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('he/data_inverse_text_normalization/test_full_sentences.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_he.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/he/test_measure.py b/tests/nemo_text_processing/he/test_measure.py new file mode 100644 index 000000000..1649effa7 --- /dev/null +++ b/tests/nemo_text_processing/he/test_measure.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + inverse_normalizer_he = InverseNormalizer(lang='he', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('he/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_he.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/he/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/he/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..bce2e24b9 --- /dev/null +++ b/tests/nemo_text_processing/he/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,61 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/he/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNDate() { + input=$PROJECT_DIR/he/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNDecimal() { + input=$PROJECT_DIR/he/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + + +testITNTime() { + input=$PROJECT_DIR/he/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNMeasure() { + input=$PROJECT_DIR/he/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + + +testITNWhitelist() { + input=$PROJECT_DIR/he/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $input +} + + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/he/test_time.py b/tests/nemo_text_processing/he/test_time.py new file mode 100644 index 000000000..f3bba67b5 --- /dev/null +++ b/tests/nemo_text_processing/he/test_time.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + inverse_normalizer_en = InverseNormalizer(lang='he', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('he/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_en.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/he/test_whitelist.py b/tests/nemo_text_processing/he/test_whitelist.py new file mode 100644 index 000000000..fb14c2a58 --- /dev/null +++ b/tests/nemo_text_processing/he/test_whitelist.py @@ -0,0 +1,31 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + inverse_normalizer_he = InverseNormalizer(lang='he', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('he/data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_he.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi/__init__.py b/tests/nemo_text_processing/hi/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/tests/nemo_text_processing/hi/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_address.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_address.txt new file mode 100644 index 000000000..69447a6e0 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_address.txt @@ -0,0 +1,25 @@ +सात शून्य शून्य ओक स्ट्रीट~७०० ओक स्ट्रीट +एक एक जंगल रोड~११ जंगल रोड +तीन शून्य एक पार्क एवेन्यू~३०१ पार्क एवेन्यू +गली नंबर एक सात जीएकगढ़~गली नंबर १७ जीएकगढ़ +अदनान अपार्टमेंट फ्लैट नंबर पाँच पाँच~अदनान अपार्टमेंट फ्लैट नंबर ५५ +प्लॉट नंबर आठ बालाजी मार्केट~प्लॉट नंबर ८ बालाजी मार्केट +बूथ सात शून्य, सेक्टर आठ, चंडीगढ़~बूथ ७०, सेक्टर ८, चंडीगढ़ +दो दो दो एक सदर्न स्ट्रीट~२२२१ सदर्न स्ट्रीट +छह दो पाँच स्कूल स्ट्रीट~६२५ स्कूल स्ट्रीट +पाँच शून्य छह स्टेट रोड~५०६ स्टेट रोड +छह छह हाइफ़न चार, पार्कहर्स्ट रोड~६६-४, पार्कहर्स्ट रोड +एक चार बटा तीन, मथुरा रोड~१४/३, मथुरा रोड +अमरावती छह पाँच पाँच नौ तीन शून्य~अमरावती ६५५९३० +अमरावती चार छह आठ दो पाँच दो~अमरावती ४६८२५२ +शिमला, हिमाचल प्रदेश पाँच नौ तीन नौ आठ आठ~शिमला, हिमाचल प्रदेश ५९३९८८ +रांची, झारखंड सात तीन छह पाँच पाँच सात~रांची, झारखंड ७३६५५७ +कोहिमा, नागालैंड चार चार आठ तीन सात सात~कोहिमा, नागालैंड ४४८३७७ +मुंबई, महाराष्ट्र आठ तीन नौ चार आठ आठ~मुंबई, महाराष्ट्र ८३९४८८ +मुंबई, महाराष्ट्र दो नौ शून्य नौ तीन सात~मुंबई, महाराष्ट्र २९०९३७ +गांधीनगर, गुजरात आठ शून्य आठ तीन सात चार~गांधीनगर, गुजरात ८०८३७४ +रायपुर, छत्तीसगढ़ एक एक शून्य छह तीन पाँच~रायपुर, छत्तीसगढ़ ११०६३५ +भोपाल, मध्य प्रदेश सात पाँच एक दो दो पाँच~भोपाल, मध्य प्रदेश ७५१२२५ +अगरतला, त्रिपुरा नौ एक पाँच तीन शून्य पाँच~अगरतला, त्रिपुरा ९१५३०५ +लखनऊ, उत्तर प्रदेश आठ शून्य दो चार आठ एक~लखनऊ, उत्तर प्रदेश ८०२४८१ +श्रीनगर, जम्मू और कश्मीर नौ छह चार पाँच दो तीन~श्रीनगर, जम्मू और कश्मीर ९६४५२३ diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..4a7221675 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,54 @@ +चार चौके~४ चौके +छः खिलाड़ी आउट~६ खिलाड़ी आउट +वनप्लस आठ प्रो~वनप्लस ८ प्रो +पाँच चार्जर~५ चार्जर +चार ओवर में सत्रह रन~४ ओवर में १७ रन +पाँच चॉकलेट्स नौ टॉफ़िज़~५ चॉकलेट्स ९ टॉफ़िज़ +दस हजार निन्यानवे~१००९९ +एक लाख एक~१००००१ +एक सौ~१०० +तीन सौ नौ~३०९ +सात सौ अट्ठानवे~७९८ +पाँच हज़ार~५००० +आठ हज़ार चार~८००४ +नौ हज़ार सोलह~९०१६ +उन्नीस सौ बारह~१९१२ +दो हज़ार दो सौ बाईस~२२२२ +चौदह हज़ार~१४००० +अठारह हज़ार छह~१८००६ +छब्बीस हज़ार इक्कीस~२६०२१ +छियानवे हज़ार आठ सौ ग्यारह~९६८११ +चार लाख~४००००० +दो लाख दो~२००००२ +सात लाख बीस~७०००२० +नौ लाख तीन सौ इक्कीस~९००३२१ +आठ लाख पाँच हज़ार तीन सौ इक्कीस~८०५३२१ +तेईस लाख~२३००००० +पन्द्रह लाख एक~१५००००१ +सत्ताईस लाख आठ सौ बीस~२७००८२० +इक्यानवे लाख इकतीस हज़ार आठ सौ उनतीस~९१३१८२९ +तीन करोड़~३००००००० +एक करोड़ एक~१००००००१ +सात करोड़ तेरह~७०००००१३ +चार करोड़ नौ सौ ग्यारह~४००००९११ +छः करोड़ पाँच हज़ार नौ सौ ग्यारह~६०००५९११ +छः करोड़ पच्चीस हज़ार नौ सौ ग्यारह~६००२५९११ +तीन करोड़ एक लाख पच्चीस हज़ार नौ सौ ग्यारह~३०१२५९११ +दो करोड़ सत्रह लाख पच्चीस हज़ार नौ सौ ग्यारह~२१७२५९११ +तीस करोड़~३०००००००० +अट्ठानवे लाख छिहत्तर हज़ार सात सौ नवासी~९८७६७८९ +तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ~२३४५५६७ +एक करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ~१२३४५५६७ +एक करोड़ इक्कीस लाख इक्कीस हज़ार दो सौ बारह~१२१२१२१२ +एक अरब बारह करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ~११२२३४५५६७ +एक अरब दो करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ~१०२२३४५५६७ +ग्यारह अरब दो करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ~११०२२३४५५६७ +इक्यावन अरब दो करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ~५१०२२३४५५६७ +सवा सात सौ~७२५ +साढ़े सात सौ~७५० +साढ़े सात हज़ार~७५०० +सवा सात हज़ार~७२५० +डेढ़ सौ~१५० +ढाई सौ~२५० +साढ़े सोलह सौ~१६५० +सवा सोलह सौ~१६२५ diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..402361d71 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,42 @@ +छः मई~६ मई +तीस जून~३० जून +पच्चीस मार्च दो हज़ार दस~२५ मार्च, २०१० +तीस मार्च उन्नीस सौ नब्बे~३० मार्च, १९९० +मार्च तीस उन्नीस सौ नब्बे~मार्च ३०, १९९० +उन्नीस जून दो हज़ार पाँच~१९ जून, २००५ +पन्द्रह जून दो हज़ार उन्नीस~१५ जून, २०१९ +आठ जनवरी~८ जनवरी +अठारह जुलाई~१८ जुलाई +छब्बीस नवंबर~२६ नवंबर +तीन अप्रैल~३ अप्रैल +चार जनवरी~४ जनवरी +एक अक्टूबर~१ अक्टूबर +तेरह सितंबर~१३ सितंबर +मार्च दो हज़ार दस~मार्च २०१० +दस मार्च~१० मार्च +बारह दिसंबर~१२ दिसंबर +दिसंबर बारह~दिसंबर १२ +एक सितंबर~१ सितंबर +तीन फ़रवरी~३ फ़रवरी +सात जून~७ जून +सत्ताईस जुलाई दो हज़ार ग्यारह~२७ जुलाई, २०११ +जुलाई सत्ताईस~जुलाई २७ +वर्ष दो हज़ार उन्नीस~वर्ष २०१९ +सन उन्नीस सौ नब्बे~सन १९९० +उन्नीस सौ नब्बे से उन्नीस सौ इक्यानबे~१९९०-१९९१ +दो हज़ार पाँच से दो हज़ार उन्नीस~२००५-२०१९ +दो हज़ार पाँच से उन्नीस~२००५-१९ +चौंतीस सौ ईसा पूर्व~३४०० ई.पू. +उन्नीस सौ बीस ईस्वी~१९२० ई. +पच्चीस जनवरी अठारह सौ तिरेपन ईसवी~२५ जनवरी, १८५३ ई. +इकत्तीस मई उन्नीस सौ नब्बे ईसवी~३१ मई, १९९० ई. +पच्चीस ईसा पूर्व~२५ ई.पू. +मार्च की दो~मार्च २ +फ़रवरी की बीस~फ़रवरी २० +उन्नीस सौ नब्बे से उन्नीस सौ इक्यानबे ईसवी~१९९०-१९९१ ई. +दो हज़ार पाँच से दो हज़ार उन्नीस ईसा पूर्व~२००५-२०१९ ई.पू. +दसवें शताब्दी~१०वें शताब्दी +अठाहरवीं शताब्दी~१८वीं शताब्दी +एक हज़ार एकवीं शताब्दी~१००१वीं शताब्दी +एक सौ उन्नीसवां शताब्दी~११९वां शताब्दी +उन्नीस सौ बीस से छब्बीस तक~१९२०-२६ तक \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..5b8d86602 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,13 @@ +दो सौ छह दशमलव दो नौ~२०६.२९ +एक सौ एक दशमलव छह~१०१.६ +एक सौ नौ दशमलव आठ~१०९.८ +एक सौ आठ दशमलव सात पाँच~१०८.७५ +एक सौ दस दशमलव सात पाँच~११०.७५ +एक सौ दो दशमलव तीन~१०२.३ +एक सौ छह दशमलव पाँच~१०६.५ +साढ़े तीन सौ दशमलव दो दो~३५०.२२ +सवा तीन सौ दशमलव दो~३२५.२ +साढ़े चार सौ दशमलव सात पाँच~४५०.७५ +सवा चार सौ दशमलव सात पाँच~४२५.७५ +ढाई सौ दशमलव छह~२५०.६ +डेढ़ सौ दशमलव सात पाँच~१५०.७५ diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..21ceff6c6 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_fraction.txt @@ -0,0 +1,31 @@ +एक सौ नौ बटा एक सौ चौबीस~१०९/१२४ +एक सौ एक बटा दो~१०१/२ +दो सौ एक बटा दो~२०१/२ +एक सौ एक बटा चार~१०१/४ +दो सौ बटा पाँच सौ~२००/५०० +दो सौ बटा बारह~२००/१२ +एक सौ तेईस बटा एक सौ पच्चीस~१२३/१२५ +छह सौ बासठ बटा एक~६६२/१ +एक सौ पाँच बटा सात~१०५/७ +छह सौ चौवन बटा तीन~६५४/३ +एक सौ तैंतीस सही एक बटा दो~१३३ १/२ +एक सौ तैंतीस सही दो बटा तीन~१३३ २/३ +एक सही छह बटा छह~१ ६/६ +दो सही एक बटा छह~२ १/६ +तीन सही तीन बटा चार~३ ३/४ +एक सौ बीस सही तीन बटा चार~१२० ३/४ +एक सौ बीस सही पिछत्तर बटा नब्बे~१२० ७५/९० +तीन सही तीन बटा चार~३ ३/४ +सवा चौरासी~८४ १/४ +डेढ़~१ १/२ +ढाई~२ १/२ +आधा~१/२ +साढ़े~१/२ +सवा~१/४ +पौन~३/४ +पौना~३/४ +सवा पैंतीस~३५ १/४ +साढ़े चार सौ बटा दस~४५०/१० +तीन चौथाई~३/४ +दो तिहाई~२/३ +एक चौथाई~१/४ diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..21615f1c5 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,48 @@ +दो सौ छह दशमलव दो नौ ग्राम~२०६.२९ g +दो सौ छह ग्राम~२०६ g +इक्कीस दशमलव शून्य सेल्सियस~२१.० °C +इक्कीस सेल्सियस~२१ °C +बारह हज़ार तेरह दशमलव सात सात सात डेसिग्राम~१२०१३.७७७ dg +बारह हज़ार तेरह डेसिग्राम~१२०१३ dg +चार सौ उनतीस दशमलव एक कैल्विन~४२९.१ K +चार सौ उनतीस कैल्विन~४२९ K +बाईस दशमलव शून्य पाँच मिलिग्राम~२२.०५ mg +बाईस मिलिग्राम~२२ mg +नौ हज़ार दशमलव शून्य शून्य मीट्रिक टन~९०००.०० t +पच्चीस दशमलव एक किग्रा~२५.१ kg +पच्चीस किग्रा~२५ kg +बानवे हज़ार तीन सौ तिरानवे दशमलव शून्य शून्य चार मिलीमीटर~९२३९३.००४ mm +बानवे हज़ार तीन सौ तिरानवे मिलीमीटर~९२३९३ mm +सात दशमलव सात इंच~७.७ in +पाँच सौ दशमलव आठ नौ तीन माइक्रॉन~५००.८९३ µm +पाँच सौ माइक्रॉन~५०० µm +पच्चीस सौ दशमलव छः छः फुट~२५००.६६ ft +पच्चीस सौ फुट~२५०० ft +छप्पन हज़ार तीस दशमलव दो वर्गसेंटीमीटर~५६०३०.२ cm² +छप्पन हज़ार तीस वर्ग सेंटीमीटर~५६०३० cm² +छियासठ दशमलव एक एकड़~६६.१ ac +छियासठ एकड़~६६ ac +चौंतीस सौ नौ दशमलव सात पाँच क्यूबिकमिलीमीटर~३४०९.७५ mm³ +छे सौ अठारह दशमलव दो दो लीटर~६१८.२२ L +चार हज़ार दशमलव शून्य शून्य गैलन~४०००.०० gal +चार हज़ार गैलन~४००० gal +तैंतीस दशमलव तीन तीन किलोमीटर प्रति घंटा~३३.३३ km/h +चौदह हज़ार इकहत्तर दशमलव नौ नौ पिंट~१४०७१.९९ pt +बहत्तर दशमलव आठ तीन मील प्रति घंटा~७२.८३ mi/h +बहत्तर मील प्रति घंटा~७२ mi/h +पौने ग्यारह घंटे~१०.७५ h +साढ़े सात वर्ष~७.५ yr +सवा ग्यारह सौ मीटर~११२५ m +पौने चार सौ हेक्टेयर~३७५ ha +साढ़े दस घन फीट~१०.५ ft³ +पौने पांच सौ किलोमीटर~४७५ km +ढाई सौ गैलन~२५० gal +डेढ़ दर्जन~१.५ doz +साढ़े सात ऐंपीयर~७.५ A +पौने तीन हजार एकड़~२७५० ac +साढ़े बारह वर्ग माइक्रोमीटर~१२.५ µm² +ढाई महीने~२.५ mo +दो बाई दो~२x२ +दो बाई दो~२x२ +पाँच बाई पाँच~५x५ +बाईस बाई पाँच घन फीट~२२x५ ft³ diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..8821940c3 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,50 @@ +तैंतीस अल्जीरियाई दिनार~دج३३ +बारह हज़ार तेरह डॉलर~$१२०१३ +चौदह हज़ार इकहत्तर दशमलव नौ नौ बेलारूसी रूबल~br१४०७१.९९ +छे सौ अठारह चीनी युआन~元६१८ +अट्ठाईस सौ दशमलव शून्य आठ आर्मेनियाई ड्राम~֏२८००.०८ +पच्चीस सौ छः अरूबान फ्लोरिन~ƒ२५०६ +बहत्तर त्रिनिदाद और टोबैगो डॉलर~tt$७२ +छियासठ तुर्की लिरा~₺६६ +चार सौ उनतीस युगांडा शिलिंग~ush४२९ +बाईस दशमलव शून्य पाँच यूक्रेनी ग्रिव्ना~₴२२.०५ +पच्चीस वॉन~₩२५ +छप्पन हज़ार तीस वेनेजुएलन बोलिवार~bs.५६०३० +चौंतीस सौ नौ साइप्रस पाउंड~cyp३४०९ +बानवे हज़ार तीन सौ तिरानवे दशमलव शून्य शून्य चार लिलांगेनी~l९२३९३.००४ +छे सौ अठारह बहरीन दिरहम~.د.ب६१८ +दो सौ छह रुपये दो सौ छह पैसे~₹२०६.२०६ +अड़तीस रुपिया~₹३८ +इक्यानबे सौ रुपेया और दो सौ पैसा~₹९१००.२०० +नौ हज़ार दशमलव शून्य शून्य पैसे~p९०००.०० +चौदह हज़ार इकहत्तर अजरबैजानी मनात~₼१४०७१ +इकहत्तर हज़ार इकहत्तर बिटकॉइन~₿७१०७१ +बत्तीस बुरुंडी फ्रैंक~fbu३२ +पन्द्रह सौ कैमन आइलैंड्स डॉलर~ci$१५०० +छह सौ पच्चीस रुपये दो पैसे~₹६२५.२ +साढ़े सात सौ डॉलर~$७५० +सवा दो सौ यूक्रेनी ग्रिव्ना~₴२२५ +साढ़े छः लाख रुपए~₹६५०००० +सवा छः लाख अल्जीरियाई दिनार~دج६२५००० +सवा पंद्रह लाख युगांडा शिलिंग~ush१५२५००० +साढ़े पंद्रह लाख रुपए~₹१५५०००० +साढ़े पाँच हज़ार लीरा~₺५५०० +ढाई सौ यूरो~€२५० +ढाई हजार बुरुंडी फ्रैंक~fbu२५०० +ढाई करोड़ रुपए~₹२५०००००० +ढाई लाख रुपए~₹२५०००० +डेढ़ सौ यूरो~€१५० +डेढ़ हजार रुपए~₹१५०० +डेढ़ करोड़ रुपए~₹१५०००००० +डेढ़ लाख रुपए~₹१५०००० +पौने तीन सौ रुपए~₹२७५ +पौने पंद्रह सौ रुपए~₹१४७५ +पौने तीन हजार रुपए~₹२७५० +पौने पंद्रह हजार यूरो~€१४७५० +पौने पैंतालिस हजार यूरो~€४४७५० +पौने तीन लाख रुपए~₹२७५००० +पौने पंद्रह लाख रुपए~₹१४७५००० +पौने पैंतालिस लाख रुपए~₹४४७५००० +पौने तीन करोड़ रुपए~₹२७५००००० +पौने पंद्रह करोड़ रुपए~₹१४७५००००० +पौने पैंतालिस करोड़ रुपए~₹४४७५००००० diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..3a65fdff7 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,13 @@ +एक हज़ार एकवीं~१००१वीं +सौवां~१००वां +एक सौ एकवां~१०१वां +दसवां~१०वां +दसवीं~१०वीं +दसवें~१०वें +एक सौ उन्नीसवां~११९वां +एक सौ उन्नीसवीं~११९वीं +एक सौ उन्नीसवें~११९वें +अट्ठानवे सौ छब्बीसवीं~९८२६वीं +अट्ठानवेवीं~९८वीं +निन्यानवेवां~९९वां +छे सौ चालीसवीं~६४०वीं \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..3b84a333d --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt @@ -0,0 +1,28 @@ +एक एक एक एक एक एक~११११११ +पाँच शून्य शून्य शून्य एक दो~५०००१२ +एक दो तीन चार पाँच छह~१२३४५६ +चार शून्य शून्य शून्य एक शून्य~४०००१० +सात पाँच शून्य शून्य शून्य दो~७५०००२ +आठ आठ शून्य नौ नौ शून्य~८८०९९० +नौ आठ सात छह पाँच चार तीन दो एक शून्य~९८७६५४३२१० +सात शून्य एक दो तीन चार पाँच छह सात आठ~७०१२३४५६७८ +आठ आठ आठ सात सात सात छह छह छह छह~८८८७७७६६६६ +छह दो नौ शून्य एक पाँच सात तीन चार आठ~६२९०१५७३४८ +नौ नौ आठ आठ सात सात छह छह पाँच पाँच~९९८८७७६६५५ +प्लस नौ एक नौ आठ सात छह पाँच चार तीन दो एक शून्य~+९१ ९८७६५४३२१० +प्लस नौ एक सात शून्य एक दो तीन चार पाँच छह सात आठ~+९१ ७०१२३४५६७८ +प्लस नौ एक आठ आठ आठ सात सात सात छह छह छह छह~+९१ ८८८७७७६६६६ +प्लस नौ एक एक एक एक एक एक एक एक एक एक एक~+९१ ११११११११११ +शून्य दो शून्य दो चार तीन सात एक पाँच चार दो~०२०२४३७१५४२ +शून्य एक एक दो छह एक दो तीन चार पाँच छह~०११२६१२३४५६ +चार चार दो दो आठ आठ छह छह चार चार~४४२२८८६६४४ +शून्य आठ शून्य चार एक दो तीन चार पाँच छह सात~०८०४१२३४५६७ +दो दो छह छह पांच चार तीन दो एक शून्य~२२६६५४३२१० +zero one three three six two three four five six seven~०१३३६२३४५६७ +zero one three four two three two one five four eight~०१३४२३२१५४८ +एक दो तीन चार~१२३४ +पाँच शून्य शून्य नौ~५००९ +चार चार चार चार~४४४४ +सात आठ नौ एक~७८९१ +एक शून्य दो शून्य~१०२० +नौ आठ सात छह~९८७६ \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..8ec5e4df3 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,25 @@ +एक बजे सात मिनट~१:०७ +दो बजकर ग्यारह मिनट~२:११ +दो बजके इकतालीस मिनट~२:४१ +बारह बजकर चौवन मिनट~१२:५४ +ग्यारह बजे~११:०० +सात बजे~७:०० +चार बजके नौ मिनट~४:०९ +आठ बजकर पैंतालीस मिनट~८:४५ +छः बजके पाँच मिनट~६:०५ +छह बजे~६:०० +बारह पन्द्रह~१२:१५ +दस अठारह~१०:१८ +चार बजे पाँच सेकंड~४:००:०५ +नौ घंटा दो सेकंड~९:००:०२ +सोलह घंटा एक मिनट सत्ताईस सेकंड~१६:०१:२७ +दस बजकर चौवन मिनट आठ सेकंड~१०:५४:०८ +तीन मिनट उन्नीस सेकंड~००:०३:१९ +ढाई बजे~२:३० +डेढ़ बजे~१:३० +डेढ़ घंटा~१:३० +साढ़े पाँच बजे~५:३० +सवा चार बजे~४:१५ +साढ़े ग्यारह~११:३० +पौने पाँच~४:४५ +पौने तीन घंटा~२:४५ diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..68f4fd775 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,8 @@ +मास्टर निखिल तनिष~मा. निखिल तनिष +पाव~१/४ +श्रीमती ज्योत्सना~स्मि. ज्योत्सना +डॉक्टर~डॉ. +आधा कप चाय~१/२ कप चाय +श्रीमान भारत कुमार~श्री. भारत कुमार +डॉक्टर प्रशांत~डॉ. प्रशांत +कुमारी~कु. diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..ce044e7cf --- /dev/null +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,15 @@ +नींद~नींद +याहू!~याहू! +-~- +आआआ~आआआ +आकाशगंगा~आकाशगंगा +लटरपटर~लटरपटर +कच्चा-पक्का~कच्चा-पक्का +गुब्बारा~गुब्बारा +चिट्ठी~चिट्ठी +ढूंढना~ढूंढना +लोहे का!~लोहे का! +टाटा~टाटा +~ +झ~झ +संगीत~संगीत \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_address.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_address.txt new file mode 100644 index 000000000..d0554ce30 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_address.txt @@ -0,0 +1,52 @@ +700 ओक स्ट्रीट~सात सौ ओक स्ट्रीट +११ जंगल रोड~ग्यारह जंगल रोड +301 पार्क एवेन्यू~तीन सौ एक पार्क एवेन्यू +गली नंबर १७ जीएकगढ़~गली नंबर सत्रह जीएकगढ़ +अदनान अपार्टमेंट फ्लैट नंबर 55~अदनान अपार्टमेंट फ्लैट नंबर पचपन +प्लॉट नंबर ८ बालाजी मार्केट~प्लॉट नंबर आठ बालाजी मार्केट +शॉप नंबर 109 9 और 10 डिवाइडिंग रोड सेक्टर 10 फरीदाबाद~शॉप नंबर एक सौ नौ नौ और दस डिवाइडिंग रोड सेक्टर दस फरीदाबाद +बूथ ७०, सेक्टर ८, चंडीगढ़~बूथ सत्तर, सेक्टर आठ, चंडीगढ़ +७०० ओक स्ट्रीट~सात सौ ओक स्ट्रीट +625 स्कूल स्ट्रीट~छह सौ पच्चीस स्कूल स्ट्रीट +१४७० एस वाशिंगटन स्ट्रीट~एक चार सात शून्य एस वाशिंगटन स्ट्रीट +506 स्टेट रोड~पाँच सौ छह स्टेट रोड +579 ट्रॉय-शेंक्टाडी रोड~पाँच सौ उनासी ट्रॉय हाइफ़न शेंक्टाडी रोड +७८३० - ई वेटरन्स पार्कवे, कोलंबस, जी ए ३१९०९~सात आठ तीन शून्य हाइफ़न ई वेटरन्स पार्कवे, कोलंबस, जी ए तीन एक नौ शून्य नौ +66-4, पार्कहर्स्ट रोड~छियासठ हाइफ़न चार, पार्कहर्स्ट रोड +८४०/१, १०० फीट रोड, मेट्रो पिलर ५६-५७, इंदिरानगर, बैंगलोर~आठ सौ चालीस बटा एक, एक सौ फीट रोड, मेट्रो पिलर छप्पन हाइफ़न सत्तावन, इंदिरानगर, बैंगलोर +17-18, राजलक्ष्मी नगर, 7th क्रॉस स्ट्रीट, 100 फीट बाईपास रोड, वेलाचेरी, चेन्नई~सत्रह हाइफ़न अठारह, राजलक्ष्मी नगर, सेवंथ क्रॉस स्ट्रीट, एक सौ फीट बाईपास रोड, वेलाचेरी, चेन्नई +४/५ न्यू म्युनिसिपल मार्केट रोड नंबर ५ और ६ सेन्टाक्रूज़ वेस्ट~चार बटा पाँच न्यू म्युनिसिपल मार्केट रोड नंबर पाँच और छह सेन्टाक्रूज़ वेस्ट +16/17 4th फ्लोर जवाहर नगर मटरू मंदिर रोड नंबर 2~सोलह बटा सत्रह फ़ोर्थ फ्लोर जवाहर नगर मटरू मंदिर रोड नंबर दो +५/३०४ सिक्का कॉम्प्लेक्स विकास मार्ग एक्सटेंशन~पाँच बटा तीन सौ चार सिक्का कॉम्प्लेक्स विकास मार्ग एक्सटेंशन +21/2 2nd फ्लोर 1st मेन रोड गांधी नगर~इक्कीस बटा दो सेकंड फ्लोर फ़र्स्ट मेन रोड गांधी नगर +नंबर २२/१८ ३rd फ्लोर सराय बोउ अली शू मार्केट~नंबर बाईस बटा अठारह थर्ड फ्लोर सराय बोउ अली शू मार्केट +14/3, मथुरा रोड~चौदह बटा तीन, मथुरा रोड +यूनिट ३ १st फ्लोर नंबर ३७ सोलेमान खतर स्ट्रीट~यूनिट तीन फ़र्स्ट फ्लोर नंबर सैंतीस सोलेमान खतर स्ट्रीट +1st फ्लोर नंबर 52 नॉर्थ अबूज़र स्ट्रीट खान ए अंसारी स्ट्रीट शरीयती स्ट्रीट 16617~फ़र्स्ट फ्लोर नंबर बावन नॉर्थ अबूज़र स्ट्रीट खान ए अंसारी स्ट्रीट शरीयती स्ट्रीट एक छह छह एक सात +२०६ जय कॉम कॉम्प्लेक्स १st पोखरन रोड~दो सौ छह जय कॉम कॉम्प्लेक्स फ़र्स्ट पोखरन रोड +नंबर 36 2nd फ्लोर सुपर 8 फेज 1 एकबतन टाउन तेहरान 13947~नंबर छत्तीस सेकंड फ्लोर सुपर आठ फेज एक एकबतन टाउन तेहरान एक तीन नौ चार सात +२nd फ्लोर नंबर ८०८ आजादी स्ट्रीट~सेकंड फ्लोर नंबर आठ सौ आठ आजादी स्ट्रीट +2nd फ्लोर नंबर 15 बिफ़ोर कांदि स्ट्रीट नॉर्थ सोहरावर्दी स्ट्रीट 15669~सेकंड फ्लोर नंबर पंद्रह बिफ़ोर कांदि स्ट्रीट नॉर्थ सोहरावर्दी स्ट्रीट एक पाँच छह छह नौ +यूनिट ४ नंबर २५ २nd गोलहा स्ट्रीट काशनी स्ट्रीट नूर स्क्वेर~यूनिट चार नंबर पच्चीस सेकंड गोलहा स्ट्रीट काशनी स्ट्रीट नूर स्क्वेर +ईस्ट 3rd फ्लोर नंबर 70 नेक्स्ट दो तोहीद इंस्टीट्यूट परचम स्ट्रीट~ईस्ट थर्ड फ्लोर नंबर सत्तर नेक्स्ट दो तोहीद इंस्टीट्यूट परचम स्ट्रीट +३rd फ्लोर नंबर ५ हमेदन एली अपोज़िट लाले पार्क नॉर्थ कारगर स्ट्रीट~थर्ड फ्लोर नंबर पाँच हमेदन एली अपोज़िट लाले पार्क नॉर्थ कारगर स्ट्रीट +4th फ्लोर नंबर 1124 जमहोरी स्ट्रीट~फ़ोर्थ फ्लोर नंबर एक एक दो चार जमहोरी स्ट्रीट +५th फ्लोर नंबर ७/१ १३th एली शाहिद अराबली स्ट्रीट~फ़िफ्थ फ्लोर नंबर सात बटा एक थर्टींथ एली शाहिद अराबली स्ट्रीट +11, 80 फीट रोड, इंडियन ऑयल पेट्रोल पंप, कोरमंगला 6th ब्लॉक, बैंगलोर के सामने~ग्यारह, अस्सी फीट रोड, इंडियन ऑयल पेट्रोल पंप, कोरमंगला सिक्स्थ ब्लॉक, बैंगलोर के सामने +२१/११, जे ब्लॉक, ६th एवेन्यू मेन रोड, अन्ना नगर पूर्व, चेन्नई~इक्कीस बटा ग्यारह, जे ब्लॉक, सिक्स्थ एवेन्यू मेन रोड, अन्ना नगर पूर्व, चेन्नई +32A नाज़ प्लाज़ा मेरिस रोड~बत्तीस ए नाज़ प्लाज़ा मेरिस रोड +२१४ बी गोविंद पूरी स्ट्रीट नंबर २~दो सौ चौदह बी गोविंद पूरी स्ट्रीट नंबर दो +२५१३ ५३ एवेन्यू, मुंबई, महाराष्ट्र ४००००१~दो पाँच एक तीन तिरेपन एवेन्यू, मुंबई, महाराष्ट्र चार शून्य शून्य शून्य शून्य एक +अमरावती ६५५९३०~अमरावती छह पाँच पाँच नौ तीन शून्य +शिमला, हिमाचल प्रदेश 593988~शिमला, हिमाचल प्रदेश पाँच नौ तीन नौ आठ आठ +रांची, झारखंड 736557~रांची, झारखंड सात तीन छह पाँच पाँच सात +कोहिमा, नागालैंड ४४८३७७~कोहिमा, नागालैंड चार चार आठ तीन सात सात +मुंबई, महाराष्ट्र 839488~मुंबई, महाराष्ट्र आठ तीन नौ चार आठ आठ +अमरावती ४६८२५२~अमरावती चार छह आठ दो पाँच दो +गांधीनगर, गुजरात 808374~गांधीनगर, गुजरात आठ शून्य आठ तीन सात चार +मुंबई, महाराष्ट्र २९०९३७~मुंबई, महाराष्ट्र दो नौ शून्य नौ तीन सात +श्रीनगर, जम्मू और कश्मीर 964523~श्रीनगर, जम्मू और कश्मीर नौ छह चार पाँच दो तीन +रायपुर, छत्तीसगढ़ ११०६३५~रायपुर, छत्तीसगढ़ एक एक शून्य छह तीन पाँच +भोपाल, मध्य प्रदेश 751225~भोपाल, मध्य प्रदेश सात पाँच एक दो दो पाँच +अगरतला, त्रिपुरा ९१५३०५~अगरतला, त्रिपुरा नौ एक पाँच तीन शून्य पाँच +लखनऊ, उत्तर प्रदेश 802481~लखनऊ, उत्तर प्रदेश आठ शून्य दो चार आठ एक diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..050310f9f --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,161 @@ +4 चौके~चार चौके +६ खिलाड़ी आउट~छह खिलाड़ी आउट +वनप्लस 8 प्रो~वनप्लस आठ प्रो +५ चार्जर~पाँच चार्जर +4 ओवर में 17 रन~चार ओवर में सत्रह रन +५ चॉकलेट्स ९ टॉफ़िज़~पाँच चॉकलेट्स नौ टॉफ़िज़ +10099~दस हज़ार निन्यानबे +१००००१~एक लाख एक +4 छक्के 14 चौके~चार छक्के चौदह चौके +६ रन बनाए~छह रन बनाए +3 गोल मारे~तीन गोल मारे +६ रन बनाए~छह रन बनाए +3 गोल मारे~तीन गोल मारे +५ चौके~पाँच चौके +2 छक्के 12 रन~दो छक्के बारह रन +९ पॉइंट्स~नौ पॉइंट्स +4 मृत 18 घायल~चार मृत अठारह घायल +५ गोल मार~पाँच गोल मार +3 बैट्समैन 12 खिलाड़ी~तीन बैट्समैन बारह खिलाड़ी +५ हार १ ड्रॉ १७ जीत~पाँच हार एक ड्रॉ सत्रह जीत +51022345567~इक्यावन अरब दो करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ +पाठ १० श्लोक २~पाठ दस श्लोक दो +101~एक सौ एक +१०२~एक सौ दो +103~एक सौ तीन +१०४~एक सौ चार +105~एक सौ पाँच +१०६~एक सौ छह +107~एक सौ सात +१०८~एक सौ आठ +109~एक सौ नौ +११०~एक सौ दस +111~एक सौ ग्यारह +११२~एक सौ बारह +113~एक सौ तेरह +११४~एक सौ चौदह +115~एक सौ पंद्रह +११६~एक सौ सोलह +817~आठ सौ सत्रह +८१८~आठ सौ अठारह +819~आठ सौ उन्नीस +८२०~आठ सौ बीस +821~आठ सौ इक्कीस +८२२~आठ सौ बाईस +823~आठ सौ तेईस +८२४~आठ सौ चौबीस +825~आठ सौ पच्चीस +८२६~आठ सौ छब्बीस +827~आठ सौ सत्ताईस +८२८~आठ सौ अट्ठाईस +829~आठ सौ उनतीस +८३०~आठ सौ तीस +831~आठ सौ इकतीस +८३२~आठ सौ बत्तीस +833~आठ सौ तैंतीस +८३४~आठ सौ चौंतीस +835~आठ सौ पैंतीस +८३६~आठ सौ छत्तीस +737~सात सौ सैंतीस +७३८~सात सौ अड़तीस +739~सात सौ उनतालीस +७४०~सात सौ चालीस +741~सात सौ इकतालीस +७४२~सात सौ बयालीस +743~सात सौ तैंतालीस +७४४~सात सौ चौवालीस +745~सात सौ पैंतालीस +४४६~चार सौ छियालीस +447~चार सौ सैंतालीस +४४८~चार सौ अड़तालीस +449~चार सौ उनचास +४५०~चार सौ पचास +461~चार सौ इकसठ +१७५~एक सौ पचहत्तर +181~एक सौ इक्यासी +१९०~एक सौ नब्बे +191~एक सौ इक्यानबे +१९९~एक सौ निन्यानबे +1001~एक हज़ार एक +१०९९~एक हज़ार निन्यानबे +5551 केले~पाँच हज़ार पाँच सौ इक्यावन केले +५५५५२ सेब~पचपन हज़ार पाँच सौ बावन सेब +53~तिरेपन +५४~चौवन +55~पचपन +५६~छप्पन +57~सत्तावन +५८~अट्ठावन +59~उनसठ +६०~साठ +61~इकसठ +६२~बासठ +63~तिरेसठ +६४~चौंसठ +65~पैंसठ +६६~छियासठ +67~सड़सठ +६८~अड़सठ +69~उनहत्तर +७०~सत्तर +71~इकहत्तर +७२~बहत्तर +73~तिहत्तर +७४~चौहत्तर +75~पचहत्तर +७६~छिहत्तर +77~सतहत्तर +७८~अठहत्तर +79~उनासी +८०~अस्सी +81~इक्यासी +८२~बयासी +83~तिरासी +८४~चौरासी +85~पचासी +८६~छियासी +87~सत्तासी +८८~अट्ठासी +89~नवासी +९०~नब्बे +91~इक्यानबे +९२~बानबे +93~तिरानबे +९४~चौरानबे +95~पंचानबे +९६~छियानबे +97~सत्तानबे +९८~अट्ठानबे +99~निन्यानबे +१३२३~एक हज़ार तीन सौ तेईस +1345~एक हज़ार तीन सौ पैंतालीस +१३४५६~तेरह हज़ार चार सौ छप्पन +12346~बारह हज़ार तीन सौ छियालीस +१२३४५६~एक लाख तेईस हज़ार चार सौ छप्पन +872987~आठ लाख बहत्तर हज़ार नौ सौ सत्तासी +९८७६०९~नौ लाख सत्तासी हज़ार छह सौ नौ +9876789~अट्ठानबे लाख छिहत्तर हज़ार सात सौ नवासी +२३४५५६७~तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ +12345567~एक करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ +१२१२१२१२~एक करोड़ इक्कीस लाख इक्कीस हज़ार दो सौ बारह +1122345567~एक अरब बारह करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ +१०२२३४५५६७~एक अरब दो करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ +11022345567~ग्यारह अरब दो करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ +५१०२२३४५५६७~इक्यावन अरब दो करोड़ तेईस लाख पैंतालीस हज़ार पाँच सौ सड़सठ +2 पॉइंट्स 12 गोल~दो पॉइंट्स बारह गोल +०५~शून्य पाँच +01~शून्य एक +०७३~शून्य सात तीन +0001~शून्य शून्य शून्य एक +०००~शून्य शून्य शून्य +3,24,50,000~तीन करोड़ चौबीस लाख पचास हज़ार +२,१२,१५,०००~दो करोड़ बारह लाख पंद्रह हज़ार +32,450,000~तीन करोड़ चौबीस लाख पचास हज़ार +४,९९,९९,०००~चार करोड़ निन्यानबे लाख निन्यानबे हज़ार +5,50,00,000~पाँच करोड़ पचास लाख +32,45,000~बत्तीस लाख पैंतालीस हज़ार +५,५६,३२०~पाँच लाख छप्पन हज़ार तीन सौ बीस +1,23,456~एक लाख तेईस हज़ार चार सौ छप्पन +12,345~बारह हज़ार तीन सौ पैंतालीस +११,२२०~ग्यारह हज़ार दो सौ बीस +1,00,00,000~एक करोड़ \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..ac12e6af6 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_date.txt @@ -0,0 +1,39 @@ +06-05~छह मई +३१-०६~इकतीस जून +02-01~दो जनवरी +०४-०१~चार जनवरी +01-10~एक अक्टूबर +१२-०७~बारह जुलाई +०४-०३~चार मार्च +25-03-2020~पच्चीस मार्च दो हज़ार बीस +३०-०५-२०७०~तीस मई दो हज़ार सत्तर +०९-१२-२१०१~नौ दिसंबर इक्कीस सौ एक +23-08-2024~तेईस अगस्त दो हज़ार चौबीस +३ मार्च~तीन मार्च +६ मार्च, २०१०~छह मार्च दो हज़ार दस +३१ मई, १९९० ई.~इकतीस मई उन्नीस सौ नब्बे ईसवी +मार्च, २०२४~मार्च दो हज़ार चौबीस +जनवरी, १९९० ई.~जनवरी उन्नीस सौ नब्बे ईसवी +२०७०~दो हज़ार सत्तर +2024~दो हज़ार चौबीस +१२० ई. पू.~एक सौ बीस ईसा पूर्व +297-272 ई. पू.~दो सौ सत्तानबे से दो सौ बहत्तर ईसा पूर्व +३२७वीं सदी~तीन सौ सत्ताईसवीं सदी +18वीं शताब्दी~अठारहवीं शताब्दी +१९वीं दशक~उन्नीसवीं दशक +1999 में~उन्नीस सौ निन्यानबे में +१९९० का~उन्नीस सौ नब्बे का +1992 की~उन्नीस सौ बानबे की +१९६० के अभिनेता है~उन्नीस सौ साठ के अभिनेता है +1788 से~सत्रह सौ अट्ठासी से +१९५४ तक~उन्नीस सौ चौवन तक +सन 1999~सन उन्नीस सौ निन्यानबे +सन् १९२०~सन् उन्नीस सौ बीस +साल 1971~साल उन्नीस सौ इकहत्तर +सन 1999 में~सन उन्नीस सौ निन्यानबे में +सन् उन्नीस सौ बीस~सन् उन्नीस सौ बीस +सन उन्नीस सौ बीस में~सन उन्नीस सौ बीस में +१९२०-२६ तक~उन्नीस सौ बीस से छब्बीस तक +2-7-1970~दो जुलाई उन्नीस सौ सत्तर +02-07-1970~दो जुलाई उन्नीस सौ सत्तर +12-07-1970~बारह जुलाई उन्नीस सौ सत्तर \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..03b01de2f --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,24 @@ +99.99~निन्यानबे दशमलव नौ नौ +९७.०~सत्तानबे दशमलव शून्य +2563.412~दो हज़ार पाँच सौ तिरेसठ दशमलव चार एक दो +७२८६०.७०~बहत्तर हज़ार आठ सौ साठ दशमलव सात शून्य +0.008~शून्य दशमलव शून्य शून्य आठ +०.०००३~शून्य दशमलव शून्य शून्य शून्य तीन +40.0~चालीस दशमलव शून्य +८०.०~अस्सी दशमलव शून्य +1500.22~एक हज़ार पाँच सौ दशमलव दो दो +५०००.१२३५६~पाँच हज़ार दशमलव एक दो तीन पाँच छह +1000.31~एक हज़ार दशमलव तीन एक +५१४६.१७~पाँच हज़ार एक सौ छियालीस दशमलव एक सात +10000.999~दस हज़ार दशमलव नौ नौ नौ +१०००००.१७~एक लाख दशमलव एक सात +10000000.31~एक करोड़ दशमलव तीन एक +१०००००००००.२२~एक अरब दशमलव दो दो +100000000000.70~एक खरब दशमलव सात शून्य +१०००००००००००००.०००३~एक नील दशमलव शून्य शून्य शून्य तीन +1000000000000000.008~एक पद्म दशमलव शून्य शून्य आठ +१०००००००००००००००००.४१२~एक शंख दशमलव चार एक दो +१९२.१६८~एक सौ बानबे दशमलव एक छह आठ +192.168~एक सौ बानबे दशमलव एक छह आठ +99.99~निन्यानबे दशमलव नौ नौ +९९.९९~निन्यानबे दशमलव नौ नौ diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..3265724a3 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,65 @@ +gmail.com~gmail डॉट com +yahoo.com~yahoo डॉट com +hotmail.com~hotmail डॉट com +google.com~google डॉट com +kumaar.org~kumaar डॉट org +kumaar.info~kumaar डॉट info +kumar@gmail.com~kumar एट gmail डॉट com +robin@hotmail.com~robin एट hotmail डॉट com +kapil@live.com~kapil एट live डॉट com +sneha@live.com~sneha एट live डॉट com +mayank@google.com~mayank एट google डॉट com +charu@yahoo.com~charu एट yahoo डॉट com +john20@yahoo.com~john दो शून्य एट yahoo डॉट com +vivaan62@gmail.com~vivaan छह दो एट gmail डॉट com +viaan15@kumaar.com~viaan एक पाँच एट kumaar डॉट com +ltaa12@gmail.com~ltaa एक दो एट gmail डॉट com +kristen11@hotmail.com~kristen एक एक एट hotmail डॉट com +dsmith@yahoo.com~dsmith एट yahoo डॉट com +hgarza@gmail.com~hgarza एट gmail डॉट com +qhill@yahoo.com~qhill एट yahoo डॉट com +green-turner.org~green हाइफ़न turner डॉट org +sharma-badami.com~sharma हाइफ़न badami डॉट com +osborne-gross.com~osborne हाइफ़न gross डॉट com +lucero-stevenson.net~lucero हाइफ़न stevenson डॉट net +https://google.com/~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश google डॉट com फॉरवर्ड स्लैश +https://github.com/~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश github डॉट com फॉरवर्ड स्लैश +https://wikipedia.org/~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश wikipedia डॉट org फॉरवर्ड स्लैश +https://amazon.com/~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश amazon डॉट com फॉरवर्ड स्लैश +www.google.com~www डॉट google डॉट com +https://www.ndtv.com~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश www डॉट ndtv डॉट com +https://www.rbi.org.in/~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश www डॉट rbi डॉट org डॉट in फॉरवर्ड स्लैश +https://www.amity.edu~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश www डॉट amity डॉट edu +https://example.com/blog/~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश example डॉट com फॉरवर्ड स्लैश blog फॉरवर्ड स्लैश +https://example.com/about.html~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश example डॉट com फॉरवर्ड स्लैश about डॉट html +https://example.com/search.php~https कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश example डॉट com फॉरवर्ड स्लैश search डॉट php +http://ati.edu~http कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश ati डॉट edu +http://gcu.edu~http कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश gcu डॉट edu +http://pima.edu~http कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश pima डॉट edu +bamu.nic.in/~bamu डॉट nic डॉट in फॉरवर्ड स्लैश +bieap.gov.in/~bieap डॉट gov डॉट in फॉरवर्ड स्लैश +www.sharda.ac.in~www डॉट sharda डॉट ac डॉट in +C:\Users\HP\Desktop~C कोलन बैकवर्ड स्लैश Users बैकवर्ड स्लैश HP बैकवर्ड स्लैश Desktop +C:\Users\HP\Downloads~C कोलन बैकवर्ड स्लैश Users बैकवर्ड स्लैश HP बैकवर्ड स्लैश Downloads +C:\Users\HP\Documents\Zoom~C कोलन बैकवर्ड स्लैश Users बैकवर्ड स्लैश HP बैकवर्ड स्लैश Documents बैकवर्ड स्लैश Zoom +/home/desktop~फॉरवर्ड स्लैश home फॉरवर्ड स्लैश desktop +/etc/apache~फॉरवर्ड स्लैश etc फॉरवर्ड स्लैश apache +/var/www~फॉरवर्ड स्लैश var फॉरवर्ड स्लैश www +192.168.1.1~एक नौ दो डॉट एक छह आठ डॉट एक डॉट एक +10.0.0.1~एक शून्य डॉट शून्य डॉट शून्य डॉट एक +83.54.245.61~आठ तीन डॉट पाँच चार डॉट दो चार पाँच डॉट छह एक +85.189.50.24~आठ पाँच डॉट एक आठ नौ डॉट पाँच शून्य डॉट दो चार +172.16.0.1~एक सात दो डॉट एक छह डॉट शून्य डॉट एक +255.255.255.0~दो पाँच पाँच डॉट दो पाँच पाँच डॉट दो पाँच पाँच डॉट शून्य +आईपी पता है 192.168.1.1~आईपी पता है एक नौ दो डॉट एक छह आठ डॉट एक डॉट एक +आईपी एड्रेस 10.0.0.1~आईपी एड्रेस एक शून्य डॉट शून्य डॉट शून्य डॉट एक +ip address 192.168.1.1~ip address एक नौ दो डॉट एक छह आठ डॉट एक डॉट एक +ip address 10.0.0.1~ip address एक शून्य डॉट शून्य डॉट शून्य डॉट एक +report.pdf~report डॉट pdf +photo.jpg~photo डॉट jpg +data.csv~data डॉट csv +robinson.org~robinson डॉट org +anand@gmail.com~anand एट gmail डॉट com +Al₂(SO₄)₃~ए एल दो ओपन ब्रेकेट एस ओ चार क्लोज़ ब्रेकेट तीन +C₂H₄~सी दो एच चार +home/desktop~home फॉरवर्ड स्लैश desktop \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..6778978b7 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,27 @@ +99/99~निन्यानबे बटा निन्यानबे +२२ ३१/१७~बाईस और इकतीस बटा सत्रह +97/0~सत्तानबे बटा शून्य +२५६३/४१२~दो हज़ार पाँच सौ तिरेसठ बटा चार सौ बारह +72860/70~बहत्तर हज़ार आठ सौ साठ बटा सत्तर +०/८~शून्य बटा आठ +3/0~तीन बटा शून्य +४०/०~चालीस बटा शून्य +80/0~अस्सी बटा शून्य +१५००/२२~एक हज़ार पाँच सौ बटा बाईस +5000/12356~पाँच हज़ार बटा बारह हज़ार तीन सौ छप्पन +१०००/३१~एक हज़ार बटा इकतीस +5146/17~पाँच हज़ार एक सौ छियालीस बटा सत्रह +१००००/९९९~दस हज़ार बटा नौ सौ निन्यानबे +100000/17~एक लाख बटा सत्रह +१०००००००/३१~एक करोड़ बटा इकतीस +1000000000/22~एक अरब बटा बाईस +१०००००००००००/७०~एक खरब बटा सत्तर +10000000000000/3~एक नील बटा तीन +१०००००००००००००००/८~एक पद्म बटा आठ +100000000000000000/412~एक शंख बटा चार सौ बारह +२ २/७~दो और दो बटा सात +120 75/90~एक सौ बीस और पचहत्तर बटा नब्बे +१/२~आधा +१/३~तिहाई +1/4~चौथाई +3/4~तीन चौथाई \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..6afd66b7f --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,69 @@ +19 m²~उन्नीस वर्ग मीटर +१२.१९ m²~बारह दशमलव एक नौ वर्ग मीटर +20 km²~बीस वर्ग किलोमीटर +२०.७ km²~बीस दशमलव सात वर्ग किलोमीटर +2 ha~दो हेक्टेयर +२.७ ha~दो दशमलव सात हेक्टेयर +1 ac~एक एकड़ +३.८ ac~तीन दशमलव आठ एकड़ +4 m~चार मीटर +४.२ m~चार दशमलव दो मीटर +18 mi~अठारह मील +१८.५५ mi~अठारह दशमलव पाँच पाँच मील +34 in~चौंतीस इंच +३४.२ in~चौंतीस दशमलव दो इंच +40 ft~चालीस फीट +४०.३ ft~चालीस दशमलव तीन फीट +39 yd~उनतालीस यार्ड +३९.१८ yd~उनतालीस दशमलव एक आठ यार्ड +35 µm~पैंतीस माइक्रोमीटर +३५.३ µm~पैंतीस दशमलव तीन माइक्रोमीटर +5 km/hr~पाँच किलोमीटर प्रति घंटा +५.३५ km/hr~पाँच दशमलव तीन पाँच किलोमीटर प्रति घंटा +3 mi/hr~तीन मील प्रति घंटा +३.५ mi/hr~तीन दशमलव पाँच मील प्रति घंटा +25 °C~पच्चीस डिग्री सेल्सियस +२५.४ °C~पच्चीस दशमलव चार डिग्री सेल्सियस +22 °F~बाईस डिग्री फारेनहाइट +२२.५ °F~बाईस दशमलव पाँच डिग्री फारेनहाइट +50 ml~पचास मिलीलीटर +५०.५ ml~पचास दशमलव पाँच मिलीलीटर +19 qt~उन्नीस क्वार्ट +१९.७ qt~उन्नीस दशमलव सात क्वार्ट +5 gal~पाँच गैलन +५.७ gal~पाँच दशमलव सात गैलन +76 pt~छिहत्तर पिंट +७६.८८ pt~छिहत्तर दशमलव आठ आठ पिंट +77 g~सतहत्तर ग्राम +७७.१९ g~सतहत्तर दशमलव एक नौ ग्राम +5 kg~पाँच किलोग्राम +५.६ kg~पाँच दशमलव छह किलोग्राम +50 kg~पचास किलोग्राम +५०.५ kg~पचास दशमलव पाँच किलोग्राम +90 mg~नब्बे मिलीग्राम +९०.७ mg~नब्बे दशमलव सात मिलीग्राम +82 cg~बयासी सेंटीग्राम +८२.५ cg~बयासी दशमलव पाँच सेंटीग्राम +97 dg~सत्तानबे डेसीग्राम +९७.७७ dg~सत्तानबे दशमलव सात सात डेसीग्राम +65 t~पैंसठ टन +६५.६ t~पैंसठ दशमलव छह टन +88 st~अट्ठासी स्टोन +८८.५ st~अट्ठासी दशमलव पाँच स्टोन +93 lb~तिरानबे पाउंड +९३.४ lb~तिरानबे दशमलव चार पाउंड +99 oz~निन्यानबे आउन्स +९९.५ oz~निन्यानबे दशमलव पाँच आउन्स +85 q~पचासी क्विंटल +८५.९९ q~पचासी दशमलव नौ नौ क्विंटल +200x10 के गद्दे~दो सौ बाई दस के गद्दे +५x५ का सोफ़ा~पाँच बाई पाँच का सोफ़ा +2x2 रुबिक्स क्यूब~दो बाई दो रुबिक्स क्यूब +१३x१३ का घर~तेरह बाई तेरह का घर +1000 yr~एक हज़ार वर्ष +९९९९ yr~नौ हज़ार नौ सौ निन्यानबे वर्ष +16.07 yr~सोलह दशमलव शून्य सात वर्ष +५ yr~पाँच साल +1.5 yr~डेढ़ साल +२.५ yr~ढाई साल +3.5 yr~साढ़े तीन साल \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..e5f157872 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_money.txt @@ -0,0 +1,144 @@ +₹111~एक सौ ग्यारह रुपए +£१८००~एक हज़ार आठ सौ पाउंड +₩760~सात सौ साठ वॉन +$५००~पाँच सौ डॉलर +₹22~बाईस रुपए +₺५६००~पाँच हज़ार छह सौ लीरा +₺1200~एक हज़ार दो सौ लीरा +₺१२४~एक सौ चौबीस लीरा +$696~छह सौ छियानबे डॉलर +₹१४४~एक सौ चौवालीस रुपए +₺615~छह सौ पंद्रह लीरा +₩३३०~तीन सौ तीस वॉन +£700~सात सौ पाउंड +₹५३०~पाँच सौ तीस रुपए +₺85~पचासी लीरा +₩१९०~एक सौ नब्बे वॉन +₺6~छह लीरा +₺१००~एक सौ लीरा +£20~बीस पाउंड +$५०१५~पाँच हज़ार पंद्रह डॉलर +₺670~छह सौ सत्तर लीरा +$८००~आठ सौ डॉलर +$75000~पचहत्तर हज़ार डॉलर +$७५०~सात सौ पचास डॉलर +₹2132~दो हज़ार एक सौ बत्तीस रुपए +₹१९८~एक सौ अट्ठानबे रुपए +₹1115~एक हज़ार एक सौ पंद्रह रुपए +₺५३०~पाँच सौ तीस लीरा +₺5~पाँच लीरा +₹३१८०~तीन हज़ार एक सौ अस्सी रुपए +₹245~दो सौ पैंतालीस रुपए +₹२१४८~दो हज़ार एक सौ अड़तालीस रुपए +₺514~पाँच सौ चौदह लीरा +₹१५७४~एक हज़ार पाँच सौ चौहत्तर रुपए +$1500~एक हज़ार पाँच सौ डॉलर +₹२७५~दो सौ पचहत्तर रुपए +₺23~तेईस लीरा +₺४०~चालीस लीरा +₺291~दो सौ इक्यानबे लीरा +₩३२~बत्तीस वॉन +$540~पाँच सौ चालीस डॉलर +$१९४६~एक हज़ार नौ सौ छियालीस डॉलर +₹650~छह सौ पचास रुपए +₺४९~उनचास लीरा +₹2190~दो हज़ार एक सौ नब्बे रुपए +₹१०००~एक हज़ार रुपए +£100~एक सौ पाउंड +₹५१३५~पाँच हज़ार एक सौ पैंतीस रुपए +₹322~तीन सौ बाईस रुपए +$७~सात डॉलर +₩195~एक सौ पंचानबे वॉन +$१०००~एक हज़ार डॉलर +₺1708~एक हज़ार सात सौ आठ लीरा +$९~नौ डॉलर +$440~चार सौ चालीस डॉलर +$१२८~एक सौ अट्ठाईस डॉलर +₺8~आठ लीरा +£३०~तीस पाउंड +₹1000~एक हज़ार रुपए +₩५४३~पाँच सौ तैंतालीस वॉन +₹405~चार सौ पाँच रुपए +£४०~चालीस पाउंड +₹47~सैंतालीस रुपए +$६९~उनहत्तर डॉलर +₹220~दो सौ बीस रुपए +₹५१~इक्यावन रुपए +₺45~पैंतालीस लीरा +₹६३३~छह सौ तैंतीस रुपए +$1000~एक हज़ार डॉलर +$२०००~दो हज़ार डॉलर +₹320~तीन सौ बीस रुपए +₹४६७~चार सौ सड़सठ रुपए +₹811~आठ सौ ग्यारह रुपए +₹१०४०~एक हज़ार चालीस रुपए +$765~सात सौ पैंसठ डॉलर +₩५९९~पाँच सौ निन्यानबे वॉन +₹550~पाँच सौ पचास रुपए +₹६५६००~पैंसठ हज़ार छह सौ रुपए +$29~उनतीस डॉलर +₩६००~छह सौ वॉन +₹189~एक सौ नवासी रुपए +₹५९१~पाँच सौ इक्यानबे रुपए +₹1689~एक हज़ार छह सौ नवासी रुपए +₹१०९~एक सौ नौ रुपए +₺12~बारह लीरा +₹२०३~दो सौ तीन रुपए +₹635~छह सौ पैंतीस रुपए +₹४६९~चार सौ उनहत्तर रुपए +$127~एक सौ सत्ताईस डॉलर +$८५~पचासी डॉलर +₹55000000~पाँच करोड़ पचास लाख रुपए +$२८२१~दो हज़ार आठ सौ इक्कीस डॉलर +₹1254000~बारह लाख चौवन हज़ार रुपए +₹३१५~तीन सौ पंद्रह रुपए +₹2044~दो हज़ार चौवालीस रुपए +₹१००००~दस हज़ार रुपए +₹545~पाँच सौ पैंतालीस रुपए +₹१८४५~एक हज़ार आठ सौ पैंतालीस रुपए +₹372~तीन सौ बहत्तर रुपए +$९८~अट्ठानबे डॉलर +₹123.57~एक सौ तेईस रुपए सत्तावन पैसे +₹९९९.५०~नौ सौ निन्यानबे रुपए पचास पैसे +£150.29~एक सौ पचास पाउंड उनतीस पेंस +£८०.३१~अस्सी पाउंड इकतीस पेंस +₩2345.10~दो हज़ार तीन सौ पैंतालीस वॉन दस जिओन +₩१००.२५~एक सौ वॉन पच्चीस जिओन +$125.70~एक सौ पच्चीस डॉलर सत्तर सेंट +$९.९९~नौ डॉलर निन्यानबे सेंट +₺80.36~अस्सी लीरा छत्तीस कुरस +₺१२३४.७८~एक हज़ार दो सौ चौंतीस लीरा अठहत्तर कुरस +৳100.42~एक सौ टका बयालीस पैसे +৳३०२५.८७~तीन हज़ार पच्चीस टका सत्तासी पैसे +¥100.48~एक सौ येन अड़तालीस सेन +¥७७७.२३~सात सौ सतहत्तर येन तेईस सेन +₦876.53~आठ सौ छिहत्तर नाइरा तिरेपन कोबो +₦१०.२७~दस नाइरा सत्ताईस कोबो +€200.90~दो सौ यूरो नब्बे सेंट +€१२३४.७५~एक हज़ार दो सौ चौंतीस यूरो पचहत्तर सेंट +$1.12~एक डॉलर बारह सेंट +$1.123~एक दशमलव एक दो तीन डॉलर +$1.1234~एक दशमलव एक दो तीन चार डॉलर +₹2.2000~दो रुपए बीस पैसे +$1.2000~एक डॉलर बीस सेंट +₹1.500~एक रुपया पचास पैसे +₹5.00~पाँच रुपए +₹१~एक रुपया +₹२.१२३~दो दशमलव एक दो तीन रुपए +₹१.१२३४~एक दशमलव एक दो तीन चार रुपए +₹3,24,50,000~तीन करोड़ चौबीस लाख पचास हज़ार रुपए +₹32,450,000~तीन करोड़ चौबीस लाख पचास हज़ार रुपए +₹5,50,00,000~पाँच करोड़ पचास लाख रुपए +₹12,54,000~बारह लाख चौवन हज़ार रुपए +₹1,00,000~एक लाख रुपए +₹2,148~दो हज़ार एक सौ अड़तालीस रुपए +₹99,999~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए +₹३,२४,५०,०००~तीन करोड़ चौबीस लाख पचास हज़ार रुपए +₹३२,४५०,०००~तीन करोड़ चौबीस लाख पचास हज़ार रुपए +₹२,१२,१५,०००~दो करोड़ बारह लाख पंद्रह हज़ार रुपए +₹५,५०,००,०००~पाँच करोड़ पचास लाख रुपए +₹१२,५४,०००~बारह लाख चौवन हज़ार रुपए +₹५,५६,३२०~पाँच लाख छप्पन हज़ार तीन सौ बीस रुपए +₹१,००,०००~एक लाख रुपए +₹२,१४८~दो हज़ार एक सौ अड़तालीस रुपए +₹९९,९९९~निन्यानबे हज़ार नौ सौ निन्यानबे रुपए \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..7c0789404 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,72 @@ +1ला~पहला +१ली~पहली +2रा~दूसरा +२री~दूसरी +3रा~तीसरा +३री~तीसरी +4था~चौथा +४थी~चौथी +5वां~पाँचवां +५वीं~पाँचवीं +6ठा~छठा +६ठी~छठी +7वां~सातवां +७वीं~सातवीं +8वां~आठवां +८वीं~आठवीं +9वां~नौवां +९वीं~नौवीं +11वां~ग्यारहवां +१२वीं~बारहवीं +14वां~चौदहवां +१६वीं~सोलहवीं +17वां~सत्रहवां +१८वीं~अठारहवीं +19वां~उन्नीसवां +२०वां~बीसवां +21वां~इक्कीसवां +२५वीं~पच्चीसवीं +27वें~सत्ताईसवें +३०वीं~तीसवीं +33वां~तैंतीसवां +४०वीं~चालीसवीं +45वां~पैंतालीसवां +५०वां~पचासवां +56वें~छप्पनवें +६०वां~साठवां +67वीं~सड़सठवीं +७५वीं~पचहत्तरवीं +80वें~अस्सीवें +८८वां~अट्ठासीवां +91वीं~इक्यानबेवीं +९९वां~निन्यानबेवां +100वां~एक सौवां +१०१वां~एक सौ एकवां +111वीं~एक सौ ग्यारहवीं +१२५वें~एक सौ पच्चीसवें +153वीं~एक सौ तिरेपनवीं +२००वीं~दो सौवीं +219वीं~दो सौ उन्नीसवीं +२४०वां~दो सौ चालीसवां +329वां~तीन सौ उनतीसवां +३६५वां~तीन सौ पैंसठवां +455वां~चार सौ पचपनवां +५५५वीं~पाँच सौ पचपनवीं +640वीं~छह सौ चालीसवीं +८९०वां~आठ सौ नब्बेवां +1001वीं~एक हज़ार एकवीं +१०९१वें~एक हज़ार इक्यानबेवें +1782वीं~सत्रह सौ बयासीवीं +१८९०वां~एक हज़ार आठ सौ नब्बेवां +1981वीं~उन्नीस सौ इक्यासीवीं +९८२६वीं~अट्ठानबे सौ छब्बीसवीं +1st~फ़र्स्ट +2nd~सेकंड +3rd~थर्ड +4th~फ़ोर्थ +5th~फ़िफ्थ +6th~सिक्स्थ +7th~सेवंथ +8th~एटथ +9th~नाइंथ +10th~टेंथ diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..340c754ed --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt @@ -0,0 +1,23 @@ +भास्कर-II~भास्कर दो +चंद्रयान-III~चंद्रयान तीन +अग्नि-IV~अग्नि चार +श्रेणी-II~श्रेणी दो +कक्षा XII~कक्षा बारह +अध्याय IV~अध्याय चार +भाग III~भाग तीन +खंड V~खंड पाँच +विश्व युद्ध II~विश्व युद्ध दो +विश्व युद्ध-II~विश्व युद्ध दो +प्रथम पंचवर्षीय योजना-I~प्रथम पंचवर्षीय योजना एक +राष्ट्रीय राजमार्ग-IV~राष्ट्रीय राजमार्ग चार +रोहिणी आर एस-I~रोहिणी आर एस एक +पीएसएलवी सी-IV~पीएसएलवी सी चार +ISRO मिशन-III~ISRO मिशन तीन +कक्षा XII की परीक्षा~कक्षा बारह की परीक्षा +XIIवीं कक्षा की परीक्षा~बारहवीं कक्षा की परीक्षा +भाग II का सारांश~भाग दो का सारांश +अध्याय IV के प्रश्न~अध्याय चार के प्रश्न +IVथी कक्षा के विद्यार्थी~चौथी कक्षा के विद्यार्थी +XC विद्यार्थी~नब्बे विद्यार्थी +LIII वा गणतंत्र दिन~तिरपन वा गणतंत्र दिन +भाग-XCIX~भाग निन्यानवे \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_serial.txt new file mode 100644 index 000000000..4a105554d --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_serial.txt @@ -0,0 +1,31 @@ +कोविड-19~कोविड-उन्नीस +कोविड-१९~कोविड-उन्नीस +5जी~पाँच जी +५जी~पाँच जी +2^2~दो स्क्वेर्ड +२^२~दो स्क्वेर्ड +1-800-555~एक-आठ सौ-पाँच सौ पचपन +3जी~तीन जी +4जी~चार जी +कोरोना-2~कोरोना-दो +अग्नि-5~अग्नि-पाँच +ओमिक्रॉन-2~ओमिक्रॉन-दो +3^2~तीन स्क्वेर्ड +2^3~दो क्यूब +5^3~पाँच क्यूब +४^५~चार टु द पावर पाँच +99-1~निन्यानबे-एक +10-20-30~दस-बीस-तीस +1-800-999~एक-आठ सौ-नौ सौ निन्यानबे +पृथ्वी-4~पृथ्वी-चार +ब्रह्मोस-1~ब्रह्मोस-एक +Q1~क्यू एक +A10~ए दस +A12~ए बारह +B-60~बी-साठ +ABC-123~ए बी सी-एक सौ तेईस +FY2024~एफ वाई दो शून्य दो चार +H2O~एच दो ओ +CO2~सी ओ दो +ABCDE1234F~ए बी सी डी ई एक दो तीन चार एफ +F16~एफ सोलह \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..4c86259e6 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_telephone.txt @@ -0,0 +1,25 @@ +मेरा पुराना नंबर था 9157114007~मेरा पुराना नंबर था शून्य नौ एक पाँच सात एक एक चार शून्य शून्य सात +इसपे कॉल करो ०३८६२-३५१७९१~इसपे कॉल करो शून्य तीन आठ छह दो तीन पाँच एक सात नौ एक +मेरे इस नंबर पे कॉल करो 1374-309988~मेरे इस नंबर पे कॉल करो शून्य एक तीन सात चार तीन शून्य नौ नौ आठ आठ +इसपे कॉल करो ०१६८९११-४५७३~इसपे कॉल करो शून्य एक छह आठ नौ एक एक चार पाँच सात तीन ++91 7440431083 मेरे इस नंबर पे कॉल करो~प्लस नौ एक सात चार चार शून्य चार तीन एक शून्य आठ तीन मेरे इस नंबर पे कॉल करो ++९१ ९२१०५१५६०६ मेरे इस नंबर पे कॉल करो~प्लस नौ एक नौ दो एक शून्य पाँच एक पाँच छह शून्य छह मेरे इस नंबर पे कॉल करो +भुगतान के लिए कार्ड के आखिरी अंक 1234 दर्ज करें~भुगतान के लिए कार्ड के आखिरी अंक एक दो तीन चार दर्ज करें +मेरा पिन कोड ११००२३ है~मेरा पिन कोड एक एक शून्य शून्य दो तीन है +मेरा पुराना नंबर था 9157114007~मेरा पुराना नंबर था शून्य नौ एक पाँच सात एक एक चार शून्य शून्य सात +इसपे कॉल करो 03862-351791~इसपे कॉल करो शून्य तीन आठ छह दो तीन पाँच एक सात नौ एक +मेरे इस नंबर पे कॉल करो 1374 309988~मेरे इस नंबर पे कॉल करो शून्य एक तीन सात चार तीन शून्य नौ नौ आठ आठ +इसपे कॉल करो 0168911-4573~इसपे कॉल करो शून्य एक छह आठ नौ एक एक चार पाँच सात तीन ++91 7440431083 मेरे इस नंबर पे कॉल करो~प्लस नौ एक सात चार चार शून्य चार तीन एक शून्य आठ तीन मेरे इस नंबर पे कॉल करो ++91 9210515606 मेरे इस नंबर पे कॉल करो~प्लस नौ एक नौ दो एक शून्य पाँच एक पाँच छह शून्य छह मेरे इस नंबर पे कॉल करो +भुगतान के लिए कार्ड के आखिरी अंक 1234 दर्ज करें~भुगतान के लिए कार्ड के आखिरी अंक एक दो तीन चार दर्ज करें +मेरा पिन कोड 110023 है~मेरा पिन कोड एक एक शून्य शून्य दो तीन है ++1 9210515606 मेरे इस नंबर पे कॉल करो~प्लस एक नौ दो एक शून्य पाँच एक पाँच छह शून्य छह मेरे इस नंबर पे कॉल करो ++४९ ९२१०५१५६०६ मेरे इस नंबर पे कॉल करो~प्लस चार नौ नौ दो एक शून्य पाँच एक पाँच छह शून्य छह मेरे इस नंबर पे कॉल करो ++353 9210515606 मेरे इस नंबर पे कॉल करो~प्लस तीन पाँच तीन नौ दो एक शून्य पाँच एक पाँच छह शून्य छह मेरे इस नंबर पे कॉल करो ++91 9876543210 123~प्लस नौ एक नौ आठ सात छह पाँच चार तीन दो एक शून्य एक दो तीन ++1 6234517890 123~प्लस एक छह दो तीन चार पाँच एक सात आठ नौ शून्य एक दो तीन ++९१ ९८७६५४३२१० १२३~प्लस नौ एक नौ आठ सात छह पाँच चार तीन दो एक शून्य एक दो तीन +(02229) 411128~शून्य दो दो दो नौ चार एक एक एक दो आठ +०२२.२९४१११२८~शून्य दो दो दो नौ चार एक एक एक दो आठ +0 (80) 26411128~शून्य आठ शून्य दो छह चार एक एक एक दो आठ \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..5bc796209 --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_time.txt @@ -0,0 +1,18 @@ +12:10:10~बारह बजकर दस मिनट दस सेकंड +५:१२:०१~पाँच बजकर बारह मिनट एक सेकंड +3:18:43~तीन बजकर अठारह मिनट तैंतालीस सेकंड +२:१६~दो बजकर सोलह मिनट +7:21~सात बजकर इक्कीस मिनट +११:५७~ग्यारह बजकर सत्तावन मिनट +8:00~आठ बजे +४:००~चार बजे +9:00~नौ बजे +सुबह के ५:२०:१२~सुबह के पाँच बजकर बीस मिनट बारह सेकंड +सुबह के 6:39~सुबह के छह बजकर उनतालीस मिनट +सुबह के २:००~सुबह के दो बजे +दोपहर के 3:59:36~दोपहर के तीन बजकर उनसठ मिनट छत्तीस सेकंड +दोपहर के १:३६~दोपहर के एक बजकर छत्तीस मिनट +दोपहर के 3:00~दोपहर के तीन बजे +रात के १०:४८:५०~रात के दस बजकर अड़तालीस मिनट पचास सेकंड +रात के 11:50~रात के ग्यारह बजकर पचास मिनट +रात के ८:००~रात के आठ बजे \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..68888de9e --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,14 @@ +डॉ. अमित~डॉक्टर अमित +प्रो. पूजा~प्रोफेसर पूजा +इं. रेखा~इंजीनियर रेखा +ले. रोहित~लेफ्टिनेंट रोहित +वै. निखिल~वैज्ञानिक निखिल +कु. मेघा~कुमारी मेघा +मा. अंकित~मास्टर अंकित +डॉ. सीमा~डॉक्टर सीमा +प्रो. अरुण~प्रोफेसर अरुण +इं. सुमित~इंजीनियर सुमित +ले. अंजलि~लेफ्टिनेंट अंजलि +वै. प्रिया~वैज्ञानिक प्रिया +कु. रिया~कुमारी रिया +मा. विवेक~मास्टर विवेक \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..e7a284e9f --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_word.txt @@ -0,0 +1,20 @@ +नींद~नींद +याहू!~याहू! +।~। +आआआ~आआआ +आकाशगंगा~आकाशगंगा +लटरपटर~लटरपटर +कच्चा-पक्का~कच्चा-पक्का +गुब्बारा~गुब्बारा +चिट्ठी~चिट्ठी +ढूंढना~ढूंढना +लोहे का!~लोहे का! +वाह!~वाह! +टाटा~टाटा +~ +झ~झ +संगीत~संगीत +This is a sentence.~This is a sentence. +google~google +mera email hai~mera email hai +user@~user@ \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/test_address.py b/tests/nemo_text_processing/hi/test_address.py new file mode 100644 index 000000000..f01dc76c3 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_address.py @@ -0,0 +1,42 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestAddress: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_address.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=True) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_address.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_cardinal.py b/tests/nemo_text_processing/hi/test_cardinal.py new file mode 100644 index 000000000..8298ec0e3 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_cardinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi/test_date.py b/tests/nemo_text_processing/hi/test_date.py new file mode 100644 index 000000000..df12e9874 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_date.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_decimal.py b/tests/nemo_text_processing/hi/test_decimal.py new file mode 100644 index 000000000..582b59422 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_decimal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_electronic.py b/tests/nemo_text_processing/hi/test_electronic.py new file mode 100644 index 000000000..6b2e3b4f0 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_electronic.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestElectronic: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=True) + assert pred == expected diff --git a/tests/nemo_text_processing/hi/test_fraction.py b/tests/nemo_text_processing/hi/test_fraction.py new file mode 100644 index 000000000..bedf9d0f7 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_fraction.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFraction: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_measure.py b/tests/nemo_text_processing/hi/test_measure.py new file mode 100644 index 000000000..71352cdc8 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_measure.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_money.py b/tests/nemo_text_processing/hi/test_money.py new file mode 100644 index 000000000..0665146a6 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_money.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_ordinal.py b/tests/nemo_text_processing/hi/test_ordinal.py new file mode 100644 index 000000000..3e5f4bfbb --- /dev/null +++ b/tests/nemo_text_processing/hi/test_ordinal.py @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi/test_roman.py b/tests/nemo_text_processing/hi/test_roman.py new file mode 100644 index 000000000..041b88fd1 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_roman.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestRoman: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_roman.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/zh/test_preprocess.py b/tests/nemo_text_processing/hi/test_serial.py similarity index 61% rename from tests/nemo_text_processing/zh/test_preprocess.py rename to tests/nemo_text_processing/hi/test_serial.py index f817517b5..43da54b17 100644 --- a/tests/nemo_text_processing/zh/test_preprocess.py +++ b/tests/nemo_text_processing/hi/test_serial.py @@ -1,30 +1,33 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized - -from ..utils import CACHE_DIR, parse_test_case_file - - -class TestPreprocess: - normalizer_zh = Normalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') - - @parameterized.expand(parse_test_case_file('zh/data_text_normalization/test_cases_preprocess.txt')) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_norm_preprocess(self, test_input, expected): - preds = self.normalizer_zh.normalize(test_input) - assert expected == preds +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestSerial: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_serial.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=True) + assert pred == expected diff --git a/tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..0e31a1a00 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,88 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + + +testITNDecimal() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNOrdinal() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} + + +testITNFraction() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_fraction.txt + runtest $input +} + + +testITNDate() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNTime() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNMeasure() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + +testITNAddress() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_address.txt + runtest $input +} + +testITNMoney() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_money.txt + runtest $input +} + +testITNTelephone() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_telephone.txt + runtest $input +} + +testITNWord() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_word.txt + runtest $input +} + +testITNWhiteList() { + input=$PROJECT_DIR/hi/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $input +} + + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..74e1cf9c9 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh @@ -0,0 +1,121 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while IFS= read -r testcase; do + IFS='~' read -r written spoken <<< "$testcase" + + # Escape backslashes and replace non breaking space with breaking space + escaped_written=$(printf '%s' "$written" | sed 's/\\/\\\\/g') + denorm_pred=$(echo "$escaped_written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + # spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + # denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # trim white space and remove space before punctuation + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/ \([!?.]\)/\1/g')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/ \([!?.]\)/\1/g')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +#testTNSpecialText() { +# input=$PROJECT_DIR/hi/data_text_normalization/test_cases_special_text.txt +# runtest $input +#} + +testTNCardinal() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNDate() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_date.txt + runtest $input +} + +testTNDecimal() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +#testTNRange() { +# input=$PROJECT_DIR/hi/data_text_normalization/test_cases_range.txt +# runtest $input +#} + +testTNSerial() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_serial.txt + runtest $input +} + +testTNRoman() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_roman.txt + runtest $input +} + +testTNElectronic() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_electronic.txt + runtest $input +} + +testTNFraction() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_fraction.txt + runtest $input +} + +testTNMoney() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_money.txt + runtest $input +} + +testTNOrdinal() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testTNTelephone() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_telephone.txt + runtest $input +} + +testTNTime() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_time.txt + runtest $input +} + +testTNMeasure() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_measure.txt + runtest $input +} + +testTNWhitelist() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testTNWord() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_word.txt + runtest $input +} + +testTNAddress() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_address.txt + runtest $input +} + +#testTNMath() { +# input=$PROJECT_DIR/en/data_text_normalization/test_cases_math.txt +# runtest $input +#} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/hi/test_telephone.py b/tests/nemo_text_processing/hi/test_telephone.py new file mode 100644 index 000000000..7e43f7e82 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_telephone.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephone: + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_time.py b/tests/nemo_text_processing/hi/test_time.py new file mode 100644 index 000000000..402faf414 --- /dev/null +++ b/tests/nemo_text_processing/hi/test_time.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_whitelist.py b/tests/nemo_text_processing/hi/test_whitelist.py new file mode 100644 index 000000000..1e45e6a0e --- /dev/null +++ b/tests/nemo_text_processing/hi/test_whitelist.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi/test_word.py b/tests/nemo_text_processing/hi/test_word.py new file mode 100644 index 000000000..6fc5883cc --- /dev/null +++ b/tests/nemo_text_processing/hi/test_word.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + inverse_normalizer = InverseNormalizer(lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=True) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/__init__.py b/tests/nemo_text_processing/hi_en/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_address.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_address.txt new file mode 100644 index 000000000..b4461477d --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_address.txt @@ -0,0 +1,41 @@ +दिल्ली एक एक शून्य शून्य शून्य एक~दिल्ली ११०००१ +मुंबई चार शून्य शून्य शून्य शून्य एक~मुंबई ४००००१ +चेन्नई छह शून्य शून्य शून्य शून्य एक~चेन्नई ६००००१ +कोलकाता सात शून्य शून्य शून्य शून्य एक~कोलकाता ७००००१ +बेंगलुरु पाँच छह शून्य शून्य शून्य एक~बेंगलुरु ५६०००१ +सात शून्य शून्य ओक स्ट्रीट~७०० ओक स्ट्रीट +एक एक जंगल रोड~११ जंगल रोड +तीन शून्य एक पार्क एवेन्यू~३०१ पार्क एवेन्यू +गली नंबर एक सात जीएकगढ़~गली नंबर १७ जीएकगढ़ +अदनान अपार्टमेंट फ्लैट नंबर पाँच पाँच~अदनान अपार्टमेंट फ्लैट नंबर ५५ +प्लॉट नंबर आठ बालाजी मार्केट~प्लॉट नंबर ८ बालाजी मार्केट +बूथ सात शून्य, सेक्टर आठ, चंडीगढ़~बूथ ७०, सेक्टर ८, चंडीगढ़ +दो दो दो एक सदर्न स्ट्रीट~२२२१ सदर्न स्ट्रीट +छह दो पाँच स्कूल स्ट्रीट~६२५ स्कूल स्ट्रीट +पाँच शून्य छह स्टेट रोड~५०६ स्टेट रोड +छह छह हाइफ़न चार, पार्कहर्स्ट रोड~६६-४, पार्कहर्स्ट रोड +एक चार बटा तीन, मथुरा रोड~१४/३, मथुरा रोड +अमरावती छह पाँच पाँच नौ तीन शून्य~अमरावती ६५५९३० +अमरावती चार छह आठ दो पाँच दो~अमरावती ४६८२५२ +शिमला, हिमाचल प्रदेश पाँच नौ तीन नौ आठ आठ~शिमला, हिमाचल प्रदेश ५९३९८८ +रांची, झारखंड सात तीन छह पाँच पाँच सात~रांची, झारखंड ७३६५५७ +कोहिमा, नागालैंड चार चार आठ तीन सात सात~कोहिमा, नागालैंड ४४८३७७ +मुंबई, महाराष्ट्र आठ तीन नौ चार आठ आठ~मुंबई, महाराष्ट्र ८३९४८८ +मुंबई, महाराष्ट्र दो नौ शून्य नौ तीन सात~मुंबई, महाराष्ट्र २९०९३७ +गांधीनगर, गुजरात आठ शून्य आठ तीन सात चार~गांधीनगर, गुजरात ८०८३७४ +रायपुर, छत्तीसगढ़ एक एक शून्य छह तीन पाँच~रायपुर, छत्तीसगढ़ ११०६३५ +भोपाल, मध्य प्रदेश सात पाँच एक दो दो पाँच~भोपाल, मध्य प्रदेश ७५१२२५ +अगरतला, त्रिपुरा नौ एक पाँच तीन शून्य पाँच~अगरतला, त्रिपुरा ९१५३०५ +लखनऊ, उत्तर प्रदेश आठ शून्य दो चार आठ एक~लखनऊ, उत्तर प्रदेश ८०२४८१ +श्रीनगर, जम्मू और कश्मीर नौ छह चार पाँच दो तीन~श्रीनगर, जम्मू और कश्मीर ९६४५२३ +seven hundred oak street~700 oak street +eleven hundred park avenue~1100 park avenue +three hundred one main street~301 main street +two hundred and fifty four san tomas avenue~254 san tomas avenue +fourteen hundred twenty eight elm street~1428 elm street +twelve hundred and eleven east avenue~1211 east avenue +one thousand one hundred laguna court~1100 laguna court +two hundred and fifty four arques avenue~254 arques avenue +मुंबई one one zero zero zero one~मुंबई ११०००१ +चेन्नई six zero zero zero zero one~चेन्नई ६००००१ +दिल्ली one one zero zero zero one~दिल्ली ११०००१ diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..4b6011eb5 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,61 @@ +शून्य~० +एक~१ +दो~२ +तीन~३ +दस~१० +ग्यारह~११ +बारह~१२ +तेरह~१३ +चौदह~१४ +पन्द्रह~१५ +सोलह~१६ +बीस~२० +तेईस~२३ +पचास~५० +सत्तर~७० +नब्बे~९० +सौ~१०० +दो सौ~२०० +एक सौ दस~११० +तीन सौ पचास~३५० +हज़ार~१००० +एक हज़ार~१००० +दो हज़ार~२००० +दस हज़ार~१०००० +एक लाख~१००००० +दो लाख~२००००० +एक करोड़~१००००००० +पाँच करोड़~५००००००० +साढ़े तीन सौ~३५० +सवा दो सौ~२२५ +डेढ़ सौ~१५० +ढाई सौ~२५० +साढ़े तीन हज़ार~३५०० +सवा दो हज़ार~२२५० +ढाई हज़ार~२५०० +पौने चार सौ~३७५ +पौने दो सौ~१७५ +zero~zero +sixty~60 +nineteen~19 +two hundred and fifty four~254 +one hundred forty seven thousand four hundred fifty one~147451 +one million one hundred fifty six thousand one hundred seventy three~1156173 +one billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961 +minus twenty five thousand thirty seven~-25037 +minus sixty~-60 +forty six thousand six hundred sixty four~46664 +two million three~2000003 +one thousand thirteen~1013 +one thousand one~1001 +one thousand one hundred~1100 +one thousand twenty six~1026 +one thousand one hundred twenty six~1126 +eleven hundred~1100 +twenty one hundred~2100 +twenty one hundred and eleven~2111 +eleven hundred twenty one~1121 +twenty one crore ninety eight lakh thirty six thousand five hundred and ninety three~219836593 +दस and twenty~१० and 20 +सौ and one hundred~१०० and 100 +two hundred और तीन सौ~200 और ३०० diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..6f5db9407 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,37 @@ +पाँच जनवरी~५ जनवरी +दस फ़रवरी~१० फ़रवरी +पन्द्रह मार्च~१५ मार्च +बीस अप्रैल~२० अप्रैल +तीस जून~३० जून +पाँच जनवरी दो हज़ार बारह~५ जनवरी, २०१२ +दस फ़रवरी उन्नीस सौ नब्बे~१० फ़रवरी, १९९० +दो हज़ार बारह~२०१२ +उन्नीस सौ सत्तर~१९७० +दो हज़ार~२००० +उन्नीस सौ~१९०० +दो हज़ार बारह से दो हज़ार पन्द्रह~२०१२-२०१५ +पहली सदी~पहली सदी +बीसवीं सदी~२०वीं सदी +दो सौ तीन ईसा पूर्व~२०३ ई.पू. +चार सौ बीस ईसवी~४२० ई. +पन्द्रह सौ ईसवी~१५०० ई. +दो हज़ार बीस ईसवी~२०२० ई. +पन्द्रह अगस्त उन्नीस सौ सैंतालीस~१५ अगस्त, १९४७ +छब्बीस जनवरी उन्नीस सौ पचास~२६ जनवरी, १९५० +january first~january 1 +july twenty second two thousand eight~july 22, 2008 +june thirty~june 30 +july twenty fifth twenty twelve~july 25, 2012 +nineteen seventeen~1917 +twenty twelve~2012 +nineteen ninety four~1994 +two thousand three~2003 +the twenty fifth of july twenty twelve~25 july, 2012 +the fifteenth of january~15 january +february twenty fifth twenty sixteen~february 25, 2016 +november twenty fourth twenty fourteen~november 24, 2014 +january पच्चीस~january २५ +पाँच january~५ january +दस jan~१० jan +march बीस~march २० +दस march twenty twenty~१० march 2020 diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date_cased.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date_cased.txt new file mode 100644 index 000000000..96bbc7d32 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_date_cased.txt @@ -0,0 +1,70 @@ +july twenty fifth two thousand twelve~july 25, 2012 +nineteen eighties~1980s +two thousand and twenty~2020 +two thousand and nine~2009 +the twenty fifth of july twenty twelve~25 july, 2012 +the twenty fifth of july two thousand twelve~25 july, 2012 +the twenty second of july twenty twelve~22 july, 2012 +the fifteenth of january~15 january +the seventeenth of may twenty ten~17 may, 2010 +january first~january 1 +july twenty second two thousand eight~july 22, 2008 +june thirty~june 30 +july twenty fifth twenty twelve~july 25, 2012 +nineteen seventeen~1917 +twenty twelve~2012 +march sixteen sixty five~march 1665 +sixteen sixty five~1665 +july two thousand twelve~july 2012 +october nineteen oh five~october 1905 +july fifteen o six~july 1506 +the twenty fifth of july twenty twelve~25 july, 2012 +july twenty fifth twenty twelve~july 25, 2012 +july twenty fifth two thousand twelve~july 25, 2012 +july one thousand eight hundred seventy six~july 1876 +february twenty fifth twenty sixteen~february 25, 2016 +november twenty fourth twenty fourteen~november 24, 2014 +nineteen ninety four~1994 +two thousand three~2003 +one thousand eight~1008 +nineteen seventy six~1976 +june twentieth twenty fourteen~june 20, 2014 +nineteen seventy three~1973 +nineteen seventy five~1975 +eleven fifty five~1155 +July twenty fifth two thousand twelve~July 25, 2012 +Nineteen eighties~1980s +Two thousand and twenty~2020 +Two thousand and nine~2009 +The twenty fifth of july twenty twelve~25 july, 2012 +The twenty fifth of july two thousand twelve~25 july, 2012 +The twenty second of july twenty twelve~22 july, 2012 +The fifteenth of january~15 january +The fifteenth of January~15 January +The seventeenth of may twenty ten~17 may, 2010 +January first~January 1 +July twenty second two thousand eight~July 22, 2008 +June thirty~June 30 +July twenty fifth twenty twelve~July 25, 2012 +Nineteen seventeen~1917 +Twenty twelve~2012 +March sixteen sixty five~March 1665 +Sixteen sixty five~1665 +July two thousand twelve~July 2012 +October nineteen oh five~October 1905 +July fifteen o six~July 1506 +The twenty fifth of july twenty twelve~25 july, 2012 +The twenty fifth of July twenty twelve~25 July, 2012 +July twenty fifth twenty twelve~July 25, 2012 +July twenty fifth two thousand twelve~July 25, 2012 +July one thousand eight hundred seventy six~July 1876 +February twenty fifth twenty sixteen~February 25, 2016 +November twenty fourth twenty fourteen~November 24, 2014 +Nineteen ninety four~1994 +Two thousand three~2003 +One thousand eight~1008 +Nineteen seventy six~1976 +June twentieth twenty fourteen~June 20, 2014 +Nineteen seventy three~1973 +Nineteen seventy five~1975 +Eleven fifty five~1155 diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..075aebab7 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,25 @@ +एक दशमलव दो छह~१.२६ +शून्य दशमलव पाँच~०.५ +ऋण एक दशमलव दो छह~-१.२६ +दो दशमलव तीन चार~२.३४ +पाँच दशमलव शून्य एक~५.०१ +five point two million~5.2 million +one hundred sixty four point five eight thousand~164.58 thousand +four hundred million~400 million +fifty billion~50 billion +one point two five billion~1.25 billion +thirteen billion~13 billion +thirty billion~30 billion +two thousand eight hundred five point eight seven three billion~2805.873 billion +eighteen~18 +eighteen point eight five~18.85 +eighteen point five o~18.50 +eighteen point five six~18.56 +eighteen point nine~18.9 +eighteen point o five~18.05 +eighteen point one two~18.12 +eighteen point o one~18.01 +zero point two six~0.26 +point zero two~.02 +sixty point two~60.2 +minus sixty point two four zero zero~-60.2400 diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..de609c263 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_electronic.txt @@ -0,0 +1,17 @@ +a dot b c at g mail dot com~a.bc@gmail.com +a at gmail dot com~a@gmail.com +c d f at a b c dot e d u~cdf@abc.edu +a b c at g mail dot a b c~abc@gmail.abc +a b c at a b c dot com~abc@abc.com +a s d f one two three at a b c dot com~asdf123@abc.com +a one b two at a b c dot com~a1b2@abc.com +a b three dot s d d dot three at g mail dot com~ab3.sdd.3@gmail.com +h t t p colon slash slash w w w dot o u r d a i l y n e w s dot com dot s m~http://www.ourdailynews.com.sm +w w w dot c o m d a i l y n e w s dot a b slash s m~www.comdailynews.ab/sm +n vidia dot com~nvidia.com +abc at gmail dot com~abc@gmail.com +athreed at gmail dot com~athreed@gmail.com +kore dot ai~kore.ai +a at nvidia dot com~a@nvidia.com +a dot b c at nvidia dot com~a.bc@nvidia.com +a b three hyphen s d d dash three at g mail dot com~ab3-sdd-3@gmail.com diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..ce534bc21 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_fraction.txt @@ -0,0 +1,39 @@ +आठ बटा तीन~८/३ +दो बटा पाँच~२/५ +एक बटा चार~१/४ +दो सही दो बटा तीन~२ २/३ +तीन सही एक बटा चार~३ १/४ +पाँच सही तीन बटा सात~५ ३/७ +सात बटे ग्यारह~७/११ +छह बटा तेरह~६/१३ +डेढ़~१ १/२ +ढाई~२ १/२ +पाव~१/४ +एक सौ नौ बटा एक सौ चौबीस~१०९/१२४ +एक सौ एक बटा दो~१०१/२ +दो सौ एक बटा दो~२०१/२ +एक सौ एक बटा चार~१०१/४ +दो सौ बटा पाँच सौ~२००/५०० +दो सौ बटा बारह~२००/१२ +एक सौ तेईस बटा एक सौ पच्चीस~१२३/१२५ +छह सौ बासठ बटा एक~६६२/१ +एक सौ पाँच बटा सात~१०५/७ +छह सौ चौवन बटा तीन~६५४/३ +एक सौ तैंतीस सही एक बटा दो~१३३ १/२ +एक सौ तैंतीस सही दो बटा तीन~१३३ २/३ +एक सही छह बटा छह~१ ६/६ +दो सही एक बटा छह~२ १/६ +तीन सही तीन बटा चार~३ ३/४ +एक सौ बीस सही तीन बटा चार~१२० ३/४ +एक सौ बीस सही पिछत्तर बटा नब्बे~१२० ७५/९० +सवा चौरासी~८४ १/४ +आधा~१/२ +साढ़े~१/२ +सवा~१/४ +पौन~३/४ +पौना~३/४ +सवा पैंतीस~३५ १/४ +साढ़े चार सौ बटा दस~४५०/१० +तीन चौथाई~३/४ +दो तिहाई~२/३ +एक चौथाई~१/४ diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..d232013ae --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,25 @@ +दो सौ मीटर~२०० m +तीन किलोग्राम~३ kg +साढ़े तीन किलोग्राम~३.५ kg +ऋण बारह किलोग्राम~-१२ kg +पचास किलोमीटर~५० km +तीन सौ ग्राम~३०० g +सवा दो किलोग्राम~२.२५ kg +दो सौ किलोमीटर प्रति घंटा~२०० km/h +बीस डिग्री सेल्सियस~२० °C +एक सौ मीटर~१०० m +two hundred meters~200 m +three hours~3 h +two hundred kilometers per hour~200 km/h +minus sixty six kilograms~-66 kg +eight point five megawatts~8.5 mW +eight point five meters~8.5 m +eight point five two percent~8.52 % +eight point four four percent~8.44 % +five degrees celsius~5 °C +seventy two degrees fahrenheit~72 °F +two hundred seventy three kelvin~273 K +eighteen feet~18 ft +eighteen point five kilometers~18.5 km +eight hundred fifty meters~850 m +eight hundred kilograms~800 kg diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..aa9107892 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,41 @@ +दस रुपये~₹१० +सौ रुपये~₹१०० +हज़ार रुपये~₹१००० +दो सौ रुपये~₹२०० +पाँच सौ रुपये~₹५०० +दो हज़ार रुपये~₹२००० +एक लाख रुपये~₹१००००० +दस रुपये और पचास पैसे~₹१०.५० +बीस डॉलर~$२० +पचास यूरो~€५० +बहत्तर बिटकॉइन~₿७२ +ढाई सौ रुपये~₹२५० +साढ़े तीन सौ रुपये~₹३५० +सवा दो सौ रुपये~₹२२५ +पौने चार सौ रुपये~₹३७५ +ढाई करोड़ रुपये~₹२५०००००० +साढ़े तीन लाख रुपये~₹३५०००० +सवा दो लाख रुपये~₹२२५००० +पचास हज़ार रुपये~₹५०००० +two dollars~$2 +one cent~$0.01 +four united states dollars and sixty nine cents~$4.69 +seventy five dollars sixty three~$75.63 +twenty nine dollars fifty cents~$29.50 +eleven dollars and fifty one cents~$11.51 +nine hundred ninety three dollars and ninety two cents~$993.92 +four hundred sixty billion won~₩460 billion +thirty billion yen~¥30 billion +two point five billion dollars~$2.5 billion +forty five billion dollars~$45 billion +fifty million dollars~$50 million +one dollar~$1 +fifteen thousand dollars~$15000 +twenty dollar~$20 +eighteen dollars~$18 +fifteen hundred dollars~$1500 +one hundred रुपये~100 रुपये +fifty रुपये~50 रुपये +सौ डॉलर~$१०० +twenty dollars और fifty cents~$20 और $0.50 +दस rupees~१० rupees diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..94910d49c --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,26 @@ +पहला~पहला +दूसरा~दूसरा +तीसरा~तीसरा +चौथा~चौथा +दसवां~१०वां +ग्यारहवां~११वां +बीसवां~२०वां +तेईसवां~२३वां +पचासवां~५०वां +नब्बेवाँ~९०वाँ +दसवीं~१०वीं +बीसवीं~२०वीं +first~1st +second~2nd +third~3rd +fourth~4th +eleventh~11th +twelfth~12th +thirteenth~13th +twenty first~21st +twenty third~23rd +one hundredth~100th +one hundred eleventh~111th +one thousandth~1000th +forty second~42nd +seventy first~71st diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..e318fba4e --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_telephone.txt @@ -0,0 +1,39 @@ +एक एक एक एक एक एक~११११११ +पाँच शून्य शून्य शून्य एक दो~५०००१२ +एक दो तीन चार पाँच छह~१२३४५६ +चार शून्य शून्य शून्य एक शून्य~४०००१० +सात पाँच शून्य शून्य शून्य दो~७५०००२ +आठ आठ शून्य नौ नौ शून्य~८८०९९० +नौ आठ सात छह पाँच चार तीन दो एक शून्य~९८७६५४३२१० +सात शून्य एक दो तीन चार पाँच छह सात आठ~७०१२३४५६७८ +आठ आठ आठ सात सात सात छह छह छह छह~८८८७७७६६६६ +छह दो नौ शून्य एक पाँच सात तीन चार आठ~६२९०१५७३४८ +नौ नौ आठ आठ सात सात छह छह पाँच पाँच~९९८८७७६६५५ +प्लस नौ एक नौ आठ सात छह पाँच चार तीन दो एक शून्य~+९१ ९८७६५४३२१० +प्लस नौ एक सात शून्य एक दो तीन चार पाँच छह सात आठ~+९१ ७०१२३४५६७८ +प्लस नौ एक आठ आठ आठ सात सात सात छह छह छह छह~+९१ ८८८७७७६६६६ +प्लस नौ एक एक एक एक एक एक एक एक एक एक एक~+९१ ११११११११११ +शून्य दो शून्य दो चार तीन सात एक पाँच चार दो~०२०२४३७१५४२ +शून्य एक एक दो छह एक दो तीन चार पाँच छह~०११२६१२३४५६ +चार चार दो दो आठ आठ छह छह चार चार~४४२२८८६६४४ +शून्य आठ शून्य चार एक दो तीन चार पाँच छह सात~०८०४१२३४५६७ +दो दो छह छह पांच चार तीन दो एक शून्य~२२६६५४३२१० +पाँच शून्य शून्य नौ~५००९ +एक शून्य दो शून्य~१०२० +one two three one two three five six seven eight~123-123-5678 +plus nine one one two three one two three five six seven eight~+91 123-123-5678 +plus forty four one two three one two three five six seven eight~+44 123-123-5678 +o two three one two three five six seven eight~023-123-5678 +oh two three one two three five six seven eight~023-123-5678 +double oh three one two three five six seven eight~003-123-5678 +two two five dot double five dot o dot four o~225.55.0.40 +two two five dot double five dot o dot forty five~225.55.0.45 +ssn is seven double nine one two three double one three~ssn is 799-12-3113 +seven nine nine~799 +a b nine~ab9 +a b c~a b c +five w k r a three one~5wkra31 +नंबर nine eight seven six five four three two one zero~नंबर 987-654-3210 +मोबाइल one two three four five six seven eight nine zero~मोबाइल 123-456-7890 +मोबाइल nine eight seven six five four three two one zero~मोबाइल 987-654-3210 +phone एक एक एक एक एक एक एक एक एक एक~phone ११११११११११ diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..134b699b4 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,29 @@ +एक बजे~१:०० +दो बजे~२:०० +दस बजे~१०:०० +बारह बजे~१२:०० +एक बजके सात मिनट~१:०७ +चार बजे चवालीस मिनट~४:४४ +दस बजके तीस मिनट~१०:३० +तीन बजके पन्द्रह मिनट~३:१५ +एक बजके दस मिनट दो सेकंड~१:१०:०२ +साढ़े तीन बजे~३:३० +सवा चार बजे~४:१५ +पौने पाँच बजे~४:४५ +डेढ़ बजे~१:३० +ढाई बजे~२:३० +eight oclock g m t~08:00 gmt +seven a m e s t~07:00 a.m. est +two p m~02:00 p.m. +two thirty~02:30 +three o'clock~03:00 +quarter past one~01:15 +half past three~03:30 +eight fifty one~08:51 +eight forty~08:40 +eleven fifty five p m~11:55 p.m. +eleven forty five a m~11:45 a.m. +eleven forty six a m~11:46 a.m. +quarter to twelve~11:45 +set alarm at ten to eleven pm~set alarm at 10:50 p.m. +one min to one am~12:59 a.m. diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..2b22a8a78 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,17 @@ +डॉक्टर~डॉ. +श्रीमती~स्मि. +श्री~श्री. +श्रीमान~श्री. +कुमारी~कु. +मास्टर~मा. +पाव~१/४ +doctor dao~dr. dao +misses smith~mrs. smith +mister dao~mr. dao +saint george~st. george +i like for example ice cream~i like e.g. ice cream +s and p five hundred~S&P 500 +seven eleven stores~7-eleven stores +r t x~RTX +nvidia a one hundred~Nvidia A100 +c u d n n~cuDNN diff --git a/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..14977faec --- /dev/null +++ b/tests/nemo_text_processing/hi_en/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,19 @@ +~ +yahoo!~yahoo! +twenty!~20 ! +x ~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aachen's~aachen's +aadri~aadri +aahar~aahar +aahh~aahh +~ +, one~, one +, one , two , three , four~, one , two , three , four +e s three~es3 +नमस्ते~नमस्ते +भारत~भारत +दुनिया~दुनिया diff --git a/tests/nemo_text_processing/hi_en/test_address.py b/tests/nemo_text_processing/hi_en/test_address.py new file mode 100644 index 000000000..4f7dc3c51 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_address.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestAddress: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_address.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_address.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi_en/test_cardinal.py b/tests/nemo_text_processing/hi_en/test_cardinal.py new file mode 100644 index 000000000..05180a19a --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_cardinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_cardinal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_date.py b/tests/nemo_text_processing/hi_en/test_date.py new file mode 100644 index 000000000..44a1203e4 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_date.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_date_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_decimal.py b/tests/nemo_text_processing/hi_en/test_decimal.py new file mode 100644 index 000000000..51e42e7c2 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_decimal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_decimal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_electronic.py b/tests/nemo_text_processing/hi_en/test_electronic.py new file mode 100644 index 000000000..e57c846de --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_electronic.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestElectronic: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_electronic_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_fraction.py b/tests/nemo_text_processing/hi_en/test_fraction.py new file mode 100644 index 000000000..db83bd32a --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_fraction.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFraction: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_measure.py b/tests/nemo_text_processing/hi_en/test_measure.py new file mode 100644 index 000000000..7298e5187 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_measure.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_measure_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_money.py b/tests/nemo_text_processing/hi_en/test_money.py new file mode 100644 index 000000000..f61aea48c --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_money.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_money_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_ordinal.py b/tests/nemo_text_processing/hi_en/test_ordinal.py new file mode 100644 index 000000000..4af306f52 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_ordinal.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_ordinal_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/hi_en/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..0a805bbed --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,102 @@ +#! /bin/sh +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNDate() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNDecimal() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNOrdinal() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testITNTime() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNMeasure() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + +testITNMoney() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_money.txt + runtest $input +} + +testITNWhitelist() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testITNTelephone() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_telephone.txt + runtest $input +} + +testITNAddress() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_address.txt + runtest $input +} + +testITNFraction() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_fraction.txt + runtest $input +} + +testITNElectronic() { + input=$PROJECT_DIR/hi_en/data_inverse_text_normalization/test_cases_electronic.txt + runtest $input +} + +# Remove all command-line arguments +shift $# + +# Load shUnit2 +. /workspace/shunit2/shunit2 diff --git a/tests/nemo_text_processing/hi_en/test_telephone.py b/tests/nemo_text_processing/hi_en/test_telephone.py new file mode 100644 index 000000000..4c3d23640 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_telephone.py @@ -0,0 +1,41 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephone: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_telephone_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_time.py b/tests/nemo_text_processing/hi_en/test_time.py new file mode 100644 index 000000000..2f872232f --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_time.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_time_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_whitelist.py b/tests/nemo_text_processing/hi_en/test_whitelist.py new file mode 100644 index 000000000..fc4b17796 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_whitelist.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_whitelist_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hi_en/test_word.py b/tests/nemo_text_processing/hi_en/test_word.py new file mode 100644 index 000000000..345d4bf82 --- /dev/null +++ b/tests/nemo_text_processing/hi_en/test_word.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + inverse_normalizer = InverseNormalizer(lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer_cased = InverseNormalizer( + lang='hi_en', cache_dir=CACHE_DIR, overwrite_cache=False, input_case="cased" + ) + + @parameterized.expand(parse_test_case_file('hi_en/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('en/data_inverse_text_normalization/test_cases_word_cased.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_cased.inverse_normalize(test_input, verbose=False) + assert pred == expected + + @parameterized.expand(parse_test_case_file('hi/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm_hi_native(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hu/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/hu/data_text_normalization/test_cases_electronic.txt index 0a3b803c4..3d9ea6dbf 100644 --- a/tests/nemo_text_processing/hu/data_text_normalization/test_cases_electronic.txt +++ b/tests/nemo_text_processing/hu/data_text_normalization/test_cases_electronic.txt @@ -1,2 +1,8 @@ abc@def.hu~a b c kukac d e f pont hu https://www.nvidia.com~h t t p s kettőspont perjel perjel w w w pont nvidia pont com +https://www.nvidia.com.~h t t p s kettőspont perjel perjel w w w pont nvidia pont com . +@jensen~kukac j e n s e n +www.nasa.gov.~w w w pont n a s a pont gov . +www.enveedya.com.hu.~w w w pont e n v e e d y a pont com pont hu . +@jensen.me~kukac j e n s e n pont m e +@wezyr1986~kukac w e z y r egy kilenc nyolc hat \ No newline at end of file diff --git a/tests/nemo_text_processing/hu/test_cardinal.py b/tests/nemo_text_processing/hu/test_cardinal.py index ca6ed1a29..2276dbfd4 100644 --- a/tests/nemo_text_processing/hu/test_cardinal.py +++ b/tests/nemo_text_processing/hu/test_cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_date.py b/tests/nemo_text_processing/hu/test_date.py index be84c5f6b..6ca267e35 100644 --- a/tests/nemo_text_processing/hu/test_date.py +++ b/tests/nemo_text_processing/hu/test_date.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_decimal.py b/tests/nemo_text_processing/hu/test_decimal.py index 6cf69a65f..7bbc0b06f 100644 --- a/tests/nemo_text_processing/hu/test_decimal.py +++ b/tests/nemo_text_processing/hu/test_decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_electronic.py b/tests/nemo_text_processing/hu/test_electronic.py index 5d3e47f73..951e2cc4c 100644 --- a/tests/nemo_text_processing/hu/test_electronic.py +++ b/tests/nemo_text_processing/hu/test_electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_fraction.py b/tests/nemo_text_processing/hu/test_fraction.py index 0789a0318..6b1477439 100644 --- a/tests/nemo_text_processing/hu/test_fraction.py +++ b/tests/nemo_text_processing/hu/test_fraction.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_measure.py b/tests/nemo_text_processing/hu/test_measure.py index 93160a382..fac51c162 100644 --- a/tests/nemo_text_processing/hu/test_measure.py +++ b/tests/nemo_text_processing/hu/test_measure.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_money.py b/tests/nemo_text_processing/hu/test_money.py index e06e407c8..cfd8d0b54 100644 --- a/tests/nemo_text_processing/hu/test_money.py +++ b/tests/nemo_text_processing/hu/test_money.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_ordinal.py b/tests/nemo_text_processing/hu/test_ordinal.py index b083a59fc..a5fdc593d 100644 --- a/tests/nemo_text_processing/hu/test_ordinal.py +++ b/tests/nemo_text_processing/hu/test_ordinal.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/hu/test_sparrowhawk_normalization.sh index f30e53807..dee2c3632 100644 --- a/tests/nemo_text_processing/hu/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/hu/test_sparrowhawk_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/hu/test_telephone.py b/tests/nemo_text_processing/hu/test_telephone.py index 8c9486557..d674af8b5 100644 --- a/tests/nemo_text_processing/hu/test_telephone.py +++ b/tests/nemo_text_processing/hu/test_telephone.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_time.py b/tests/nemo_text_processing/hu/test_time.py index c173c02e1..1c6b56d4b 100644 --- a/tests/nemo_text_processing/hu/test_time.py +++ b/tests/nemo_text_processing/hu/test_time.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_whitelist.py b/tests/nemo_text_processing/hu/test_whitelist.py index 1fa2d2ac6..0cab00cc1 100644 --- a/tests/nemo_text_processing/hu/test_whitelist.py +++ b/tests/nemo_text_processing/hu/test_whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hu/test_word.py b/tests/nemo_text_processing/hu/test_word.py index d22e8bf11..d11092aaa 100644 --- a/tests/nemo_text_processing/hu/test_word.py +++ b/tests/nemo_text_processing/hu/test_word.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/hy/__init__.py b/tests/nemo_text_processing/hy/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/tests/nemo_text_processing/hy/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..42c015b0a --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,23 @@ +հարյուր~100 +տասներեք~13 +տասներկու~12 +քսանութ~28 +տասնհինգ հարյուր~15 100 +հիսունհինգ հազար~55000 +երկու միլիոն հարյուր իննսունինը~2000199 +մեկ միլիոն հարյուր հիսունվեց հազար հարյուր յոթանասուներեք~1156173 +մեկ միլիարդ հինգ հարյուր իննսուներեք միլիոն յոթանասուներկու հազար ինը հարյուր վաթսունմեկ~1593072961 +քսանհինգ հազար իննսունհինգ~25095 +քառասունվեց հազար վեց հարյուր~46600 +վաթսուն~60 +հազար~1000 +հարյուր~100 +հարյուր քսան~120 +հազար հարյուր հիսունյոթ~1157 +տասնութ միլիոն ինը հարյուր քառասուն հազար յոթ հարյուր քսաներկու~18940722 +տասնութ միլիոն վեց հարյուր իննսուն հազար ինը հարյուր տասնվեց~18690916 +տասնութ հազար ութ հարյուր ութսուն~18880 +հազար հարյուր~1100 +երկու հազար հարյուր տասնմեկ~2111 +վաթսուն~60 +քառասունվեց հազար վեց հարյուր վաթսունչորս~46664 diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..ac7a23cfd --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,14 @@ +զրո ամբողջ մեկ~0.1 +զրո ամբողջ յոթ~0.7 +մեկ ամբողջ յոթ միլիոնի պարտք պետությանը~1.7 միլիոնի պարտք պետությանը +մեկ ամբողջ երեք~1.3 +մեկ ամբողջ երկու~1.2 +այդ ամենը տևեց հինգ ու կես տարի~այդ ամենը տևեց 5.5 տարի +զրո ամբողջ ինը~0.9 +երկու ամբողջ երեք~2.3 +երկու ամբողջ հինգ~2.5 +մեկ ամբողջ երկու միլիարդ~1.2 միլիարդ +այստեղ ապրում է մոտ երեք ամբողջ յոթ միլիոն մարդ~այստեղ ապրում է մոտ 3.7 միլիոն մարդ +հինգ ու կես միլիոն~5.5 միլիոն +հարյուր իննսունյոթ ու կես միլիոն~197.5 միլիոն +ութ հարյուր տրիլիոն~800 տրիլիոն diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..e31640d03 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_fraction.txt @@ -0,0 +1,24 @@ +մեկ տասնմեկերորդ~1/11 +մեկ տասներկուերորդ~1/12 +մեկ տասներեքերորդ~1/13 +մեկ քառորդ~1/4 +մեկ չորրորդ~1/4 +օվկիանոսները կազմում են երկրագնդմի մակերեսի յոթանասունմեկ հարյուրերորդ մասը~օվկիանոսները կազմում են երկրագնդմի մակերեսի 71/100 մասը +մեկ հիսունհինգերորդ~1/55 +հինգ քսաներորդ~5/20 +պիցցայի մեկ երկրորդ մասը~պիցցայի 1/2 մասը +մեկ հիսուներորդ~1/50 +իննսունինը վաթսուներորդ~99/60 +մեկ յոթանասուներորդ~1/70 +որոշ բույսի չորս հինգերորդ մասը կազմված են ջրից~որոշ բույսի 4/5 մասը կազմված են ջրից +մեկ ութսուներորդ~1/80 +մեկ իննսուներորդ~1/90 +մեկ հարյուրերորդ~1/100 +երկու հինգերորդ~2/5 +տասն երեսուներորդ~10/30 +երեք քսաներորդ~3/20 +տասներկու հարյուրերորդ~12/100 +հիսունհինգ հարյուրերորդ~55/100 +հիսուն հինգ հարյուրերորդ~50/500 +մեկ տասներորդ~1/10 +մեկ քառասուներորդ~1/40 diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..93ab72d08 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,12 @@ +երկու հարյուր մետր~200 մ +հիսունվեց ամբողջ երեք քառակուսի կիլոմետր~56.3 կմ² +երկու հարյուր կիլոմետր ժամում~200 կմ/ժ +երկու հարյուր կիլոմետր ժամում~200 կմ/ժ +վաթսունվեց կիլոգրամ~66 կգ +հիսուն րոպե~50 ր +երկու հարյուր ձիաուժ~200 ձ.ու. +իննսուն գրամ~90 գ +երեք հարյուր սանտիմետր~300 սմ +վաթսունհինգ քառակուսի կիլոմետր~65 կմ² +հիսուն հերց~50 Հց +ինը հարյուր գեգաբայթ~900 գբ diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..757543476 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,10 @@ +մեկ դոլար~1 $ +մեկ ռուբլի~1 ₽ +երեսունմեկ ռուբլի~31 ₽ +տասնվեց հազար ամն դոլար~16000 $ +տասնութ հազար դոլար~18000 $ +հիսունհինգ հազար դրամ~55000 ֏ +երկուս ու կես միլիոն տենգե~2.5 միլիոն ₸ +երեք ամբողջ երկու միլիարդ լարի~3.2 միլիարդ ₾ +իննսուն հազար քսանմեկ զլոտի~90021 zł +երեք բիթքոին~3 ₿ diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..34831ee2e --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,11 @@ +տասնմեկերորդ~11-րդ +տասներկուերորդ~12-րդ +տասներեքերորդ~13-րդ +քսանմեկերորդ~21-րդ +քսաներեքերորդ~23-րդ +հարյուր տասնմեկերորդ~111-րդ +հարյուր միլիոն քսանհինգերորդ~100000025-րդ +հազարերորդ~1000-րդ +հարյուր քսանմեկերորդ~121-րդ +երկրորդ~2-րդ +առաջին~1-ին diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..9a9c4e14a --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,9 @@ +ութ անց հիսունհինգ~08:55 +երեքին հինգ պակաս~02:55 +հինգ անց կես~05:30 +քսաներեքն անց հիսունհինգ~23:55 +մեկ անց կես~01:30 +տասնմեկին տասնհինգ պակաս~10:45 +վեցին տասը պակաս~05:50 +երկուս անց քսաներկու~02:22 +յոթ անց կես~07:30 diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..4cd226d88 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,7 @@ +սույն թվականի~ս.թ. +մեր թվարկությունից առաջ~մ.թ.ա. +մեր թվարկություն~մ.թ. +քրիստոսից առաջ~Ք.ա. +քրիստոսից հետո~Ք.հ. +քրիստոսի ծնունդից առաջ~Ք.ծ.ա. +քրիստոսից ծնունդից հետո~Ք.ծ.հ. \ No newline at end of file diff --git a/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..304118a4d --- /dev/null +++ b/tests/nemo_text_processing/hy/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,50 @@ +~ +yahoo!~yahoo! +քսան !~20 ! +սսսս հիսունհինգ~սսսս 55 +x~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..eb52ea310 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,12 @@ +0~զրո +6~վեց +91~իննսունմեկ +10~տասը +13~տասներեք +83~ութսուներեք +100~հարյուր +175~հարյուր յոթանասունհինգ +312247~երեք հարյուր տասներկու հազար երկու հարյուր քառասունյոթ +1000000~մեկ միլիոն +1892004~մեկ միլիոն ութ հարյուր իննսուներկու հազար չորս +8555~ութ հազար հինգ հարյուր հիսունհինգ diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..d2a50b20d --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,9 @@ +0.2 միլիարդ~զրո ամբողջ երկու միլիարդ +30.2 միլիարդ~երեսուն ամբողջ երկու միլիարդ +390.3 միլիոն~երեք հարյուր իննսուն ամբողջ երեք միլիոն +10 միլիարդ~տասը միլիարդ +818303~ութ հարյուր տասնութ հազար երեք հարյուր երեք +24313~քսանչորս հազար երեք հարյուր տասներեք +5988005~հինգ միլիոն ինը հարյուր ութսունութ հազար հինգ +13~տասներեք +8900~ութ հազար ինը հարյուր diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..7415fceda --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,10 @@ +1/2~մեկ երկրորդ +1/3~մեկ երրորդ +1/4~մեկ չորրորդ +50/85~հիսուն ութսունհինգերորդ +1/6~մեկ վեցերորդ +1/7~մեկ յոթերորդ +1/8~մեկ ութերորդ +2/9~երկու իններորդ +1/10~մեկ տասներորդ +59/3~հիսունինը երրորդ diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..3774f1dd9 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,9 @@ +200 մ~երկու հարյուր մետր +56.3 կմ²~հիսունվեց ամբողջ երեք քառակուսի կիլոմետր +100 կմ/ժ~հարյուր կիլոմետր ժամ +95 ր~իննսունհինգ րոպե +10 կգ~տասը կիլոգրամ +90 գ~իննսուն գրամ +300 սմ~երեք հարյուր սանտիմետր +65 կմ²~վաթսունհինգ քառակուսի կիլոմետր +50 դմ~հիսուն դեցիմետր diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..0da5951dc --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_money.txt @@ -0,0 +1,10 @@ +1 $~մեկ դոլար +1 ₽~մեկ ռուբլի +31 ₽~երեսունմեկ ռուբլի +16000 $~տասնվեց հազար դոլար +18000 $~տասնութ հազար դոլար +55000 ֏~հիսունհինգ հազար դրամ +2.5 միլիոն ₸~երկու ամբողջ հինգ միլիոն տենգե +3.2 միլիարդ ₾~երեք ամբողջ երկու միլիարդ լարի +90021 zł~իննսուն հազար քսանմեկ զլոտի +3 ₿~երեք բիթքոին diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..7c59cfcc6 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,11 @@ +1-ին~առաջին +3-րդ~երրորդ +8-րդ~ութերորդ +14-րդ~տասնչորսերորդ +6-րդ~վեցերորդ +5-րդ~հինգերորդ +9-րդ~իններորդ +4-րդ~չորրորդ +100-րդ~հարյուրերորդ +1000-րդ~հազարերորդ +2540-րդ~երկու հազար հինգ հարյուր քառասուներորդ diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..d48caae3a --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_time.txt @@ -0,0 +1,9 @@ +08:55~ութ անց հիսունհինգ +02:55~երկուսն անց հիսունհինգ +05:30~հինգ անց երեսուն +23:55~քսաներեք անց հիսունհինգ +01:30~մեկ անց երեսուն +10:45~տասն անց քառասունհինգ +05:50~հինգ անց հիսուն +02:22~երկուսն անց քսաներկու +07:30~յոթ անց երեսուն diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..3994e668d --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,14 @@ +ս.թ.~սույն թվականի +մ.թ.ա.~մեր թվարկությունից առաջ +մ.թ.~մեր թվարկություն +Ք.ա.~քրիստոսից առաջ +Ք.հ.~քրիստոսից հետո +Ք.ծ.ա.~քրիստոսի ծննդից առաջ +Ք.ծ.հ.~քրիստոսի ծննդից հետո +ս․թ․~սույն թվականի +մ․թ․ա․~մեր թվարկությունից առաջ +մ․թ․~մեր թվարկություն +Ք․ա․~քրիստոսից առաջ +Ք․հ․~քրիստոսից հետո +Ք․ծ․ա․~քրիստոսի ծննդից առաջ +Ք․ծ․հ․~քրիստոսի ծննդից հետո diff --git a/tests/nemo_text_processing/hy/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..ae39d0200 --- /dev/null +++ b/tests/nemo_text_processing/hy/data_text_normalization/test_cases_word.txt @@ -0,0 +1,50 @@ +~ +yahoo!~yahoo! +20 !~քսան ! +սսսս 55~սսսս հիսունհինգ +x~x +—~— +aaa~aaa +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aagnes~aagnes +aagomoni~aagomoni +aagon~aagon +aagoo~aagoo +aagot~aagot +aahar~aahar +aahh~aahh +aahperd~aahperd +aaibinterstate~aaibinterstate +aajab~aajab +aakasa~aakasa +aakervik~aakervik +aakirkeby~aakirkeby +aalam~aalam +aalbaek~aalbaek +aaldiu~aaldiu +aalem~aalem +a'ali~a'ali +aalilaassamthey~aalilaassamthey +aalin~aalin +aaliyan~aaliyan +aaliyan's~aaliyan's +aamadu~aamadu +aamara~aamara +aambala~aambala +aamera~aamera +aamer's~aamer's +aamina~aamina +aaminah~aaminah +aamjiwnaang~aamjiwnaang diff --git a/tests/nemo_text_processing/hy/test_cardinal.py b/tests/nemo_text_processing/hy/test_cardinal.py new file mode 100644 index 000000000..74bd5170c --- /dev/null +++ b/tests/nemo_text_processing/hy/test_cardinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=True) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=True) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred in expected diff --git a/tests/nemo_text_processing/hy/test_decimal.py b/tests/nemo_text_processing/hy/test_decimal.py new file mode 100644 index 000000000..aaa65a0b7 --- /dev/null +++ b/tests/nemo_text_processing/hy/test_decimal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hy/test_fraction.py b/tests/nemo_text_processing/hy/test_fraction.py new file mode 100644 index 000000000..c9fcc7873 --- /dev/null +++ b/tests/nemo_text_processing/hy/test_fraction.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFraction: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=True) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=True) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=True, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hy/test_measure.py b/tests/nemo_text_processing/hy/test_measure.py new file mode 100644 index 000000000..9402523c7 --- /dev/null +++ b/tests/nemo_text_processing/hy/test_measure.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hy/test_money.py b/tests/nemo_text_processing/hy/test_money.py new file mode 100644 index 000000000..291ce764f --- /dev/null +++ b/tests/nemo_text_processing/hy/test_money.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hy/test_ordinal.py b/tests/nemo_text_processing/hy/test_ordinal.py new file mode 100644 index 000000000..1e93f5f2e --- /dev/null +++ b/tests/nemo_text_processing/hy/test_ordinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=True) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=True) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hy/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/hy/test_sparrowhawk_inverse_text_normalization.sh new file mode 100755 index 000000000..ebde5c9d8 --- /dev/null +++ b/tests/nemo_text_processing/hy/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,71 @@ +#! /bin/sh + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNDecimal() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNOrdinal() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testITNFraction() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_fraction.txt + runtest $input +} + +testITNTime() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNMeasure() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + +testITNMoney() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_money.txt + runtest $input +} + +testITNWhitelist() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testITNWord() { + input=$PROJECT_DIR/hy/data_inverse_text_normalization/test_cases_word.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/hy/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/hy/test_sparrowhawk_normalization.sh new file mode 100755 index 000000000..15d211c0b --- /dev/null +++ b/tests/nemo_text_processing/hy/test_sparrowhawk_normalization.sh @@ -0,0 +1,69 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while read testcase; do + IFS='~' read written spoken <<< $testcase + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNDecimal() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +testTNFraction() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_fraction.txt + runtest $input +} + +testTNOrdinal() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testTNTime() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_time.txt + runtest $input +} + +testTNMeasure() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_measure.txt + runtest $input +} + +testTNWhitelist() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testTNWord() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_word.txt + runtest $input +} + +testTNMoney() { + input=$PROJECT_DIR/hy/data_text_normalization/test_cases_money.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/hy/test_time.py b/tests/nemo_text_processing/hy/test_time.py new file mode 100644 index 000000000..6c0f72537 --- /dev/null +++ b/tests/nemo_text_processing/hy/test_time.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hy/test_whitelist.py b/tests/nemo_text_processing/hy/test_whitelist.py new file mode 100644 index 000000000..75562cf9f --- /dev/null +++ b/tests/nemo_text_processing/hy/test_whitelist.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/hy/test_word.py b/tests/nemo_text_processing/hy/test_word.py new file mode 100644 index 000000000..30f7274b1 --- /dev/null +++ b/tests/nemo_text_processing/hy/test_word.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + + inverse_normalizer = InverseNormalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('hy/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='hy', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('hy/data_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_cardinal.txt index 127addd64..795ec896e 100644 --- a/tests/nemo_text_processing/it/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_cardinal.txt @@ -8,4 +8,6 @@ 3544~tremila cinquecento quarantaquattro 1000~mille 1034500~un milione trentaquattromila cinquecento -3453243534~tremila quattrocento cinquantatre milioni duecento quarantatremila cinquecento trentaquattro \ No newline at end of file +3453243534~tremila quattrocento cinquantatre milioni duecento quarantatremila cinquecento trentaquattro +38~trentotto +7 giorni sono una settimana~sette giorni sono una settimana \ No newline at end of file diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_electronic.txt index 40b679ab1..cf011e9dd 100644 --- a/tests/nemo_text_processing/it/data_text_normalization/test_cases_electronic.txt +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_electronic.txt @@ -1,6 +1,13 @@ +Trovateci su mail.university.edu.~Trovateci su m a i l punto u n i v e r s i t y punto edu . +Trovateci su mail.nasa.gov.~Trovateci su m a i l punto n a s a punto gov . +Trovateci su mail.nasa.mx.~Trovateci su m a i l punto n a s a punto m x . https://www.nvidia.com~h t t p s due punti slash slash w w w punto nvidia punto com https://www.nvidia.com/abc/ciao.html~h t t p s due punti slash slash w w w punto nvidia punto com slash a b c slash c i a o punto html https://www.nvidia.it/abc/12df/like.py~h t t p s due punti slash slash w w w punto nvidia punto IT slash a b c slash uno due d f slash l i k e punto python abc.def@23df.sd.IT~a b c punto d e f chiocciola due tre d f punto s d punto I T abc.de!f@23d_f.sd.us~a b c punto d e punto esclamativo f chiocciola due tre d trattino basso f punto s d punto US -www.nvidia.file&TN.com~w w w punto nvidia punto f i l e e commerciale T N punto com \ No newline at end of file +www.nvidia.file&TN.com~w w w punto nvidia punto f i l e e commerciale T N punto com +@jensen~chiocciola j e n s e n +@jensen.me~chiocciola j e n s e n punto m e +@wezyr1986~chiocciola w e z y r uno nove otto sei +visitate il sito https://www.nvidia.com.~visitate il sito h t t p s due punti slash slash w w w punto nvidia punto com . diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_measure.txt index 6595ccd37..aa1e945d0 100644 --- a/tests/nemo_text_processing/it/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_measure.txt @@ -3,4 +3,5 @@ 4,3 l~quattro virgola tre litri 5 km/s~cinque chilometri per secondo 15 A~quindici ampere -155 d~cento cinquantacinque giorni \ No newline at end of file +155 d~cento cinquantacinque giorni +il 18% delle emissioni di carbonio~il diciotto percento delle emissioni di carbonio \ No newline at end of file diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_money.txt index db072f182..f800465df 100644 --- a/tests/nemo_text_processing/it/data_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_money.txt @@ -1,4 +1,5 @@ 2,01 ₽~due rubli un copeca 3,23€~tre euro ventitre centesimi 4,2 £~quattro sterline venti penny -1 eur~un euro \ No newline at end of file +1 eur~un euro +1 eur per il caffè~un euro per il caffè \ No newline at end of file diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..cc8e7667c --- /dev/null +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_time.txt @@ -0,0 +1,6 @@ +12:30~dodici e trenta minuti~dodici e mezza +05:15~cinque e quindici minuti~cinque e un quarto +17:15:26~diciassette e quindici minuti e ventisei secondi~diciassette e un quarto e ventisei secondi +23:45~ventitre e quarantacinque minuti +03:38~tre e trentotto minuti +l'evento inizia alle 16:00~l'evento inizia alle sedici \ No newline at end of file diff --git a/tests/nemo_text_processing/it/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/it/data_text_normalization/test_cases_whitelist.txt index e06971061..0fa7bd5c4 100644 --- a/tests/nemo_text_processing/it/data_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/it/data_text_normalization/test_cases_whitelist.txt @@ -1,5 +1,4 @@ -Avv.~avvocato -Sig. Luca~signore Luca +avv.~avvocato dip. di Fisica~dipartimento di Fisica sig.na Sonia~signorina Sonia p.za Grande~piazza Grande \ No newline at end of file diff --git a/tests/nemo_text_processing/it/test_cardinal.py b/tests/nemo_text_processing/it/test_cardinal.py index 7837eb3b1..87b94ee36 100644 --- a/tests/nemo_text_processing/it/test_cardinal.py +++ b/tests/nemo_text_processing/it/test_cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/it/test_decimal.py b/tests/nemo_text_processing/it/test_decimal.py index 822ef4827..c37dbcc99 100644 --- a/tests/nemo_text_processing/it/test_decimal.py +++ b/tests/nemo_text_processing/it/test_decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/it/test_electronic.py b/tests/nemo_text_processing/it/test_electronic.py index d8e116d4c..bf9390883 100644 --- a/tests/nemo_text_processing/it/test_electronic.py +++ b/tests/nemo_text_processing/it/test_electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/it/test_measure.py b/tests/nemo_text_processing/it/test_measure.py index 85f9bc0d3..4287e5b7d 100644 --- a/tests/nemo_text_processing/it/test_measure.py +++ b/tests/nemo_text_processing/it/test_measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/it/test_money.py b/tests/nemo_text_processing/it/test_money.py index a65cd2528..e98a1e942 100644 --- a/tests/nemo_text_processing/it/test_money.py +++ b/tests/nemo_text_processing/it/test_money.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/it/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/it/test_sparrowhawk_normalization.sh index 253361a07..5570c43b0 100644 --- a/tests/nemo_text_processing/it/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/it/test_sparrowhawk_normalization.sh @@ -1,17 +1,23 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do - IFS='~' read written spoken <<< $testcase - # replace non breaking space with breaking space - denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + IFS='~' read -a testcase_tokenized <<< $testcase + written=${testcase_tokenized[0]} + # only tests against first possible option when there are multiple shortest paths + spoken=${testcase_tokenized[1]} + # replace non breaking space with breaking space + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + # trim white space spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" @@ -21,6 +27,11 @@ runtest () { done < "$input" } +testTNTime() { + input=$PROJECT_DIR/it/data_text_normalization/test_cases_time.txt + runtest $input +} + testTNCardinal() { input=$PROJECT_DIR/it/data_text_normalization/test_cases_cardinal.txt runtest $input @@ -52,4 +63,4 @@ testTNMoney() { } # Load shUnit2 -. $PROJECT_DIR/../shunit2/shunit2 +. $PROJECT_DIR/../shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/test_math.py b/tests/nemo_text_processing/it/test_time.py similarity index 62% rename from tests/nemo_text_processing/zh/test_math.py rename to tests/nemo_text_processing/it/test_time.py index e8740aa2d..8bb0eb1ab 100644 --- a/tests/nemo_text_processing/zh/test_math.py +++ b/tests/nemo_text_processing/it/test_time.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,18 +13,19 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file -class TestMath: - normalizer_zh = Normalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') +class TestChar: + normalizer = Normalizer(lang='it', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') - @parameterized.expand(parse_test_case_file('zh/data_text_normalization/test_cases_math.txt')) + @parameterized.expand(parse_test_case_file('it/data_text_normalization/test_cases_time.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_norm_math(self, test_input, expected): - preds = self.normalizer_zh.normalize(test_input) - assert expected == preds + def test_norm_char(self, test_input, expected): + preds = self.normalizer.normalize(test_input, punct_post_process=True) + assert preds in expected diff --git a/tests/nemo_text_processing/it/test_whitelist.py b/tests/nemo_text_processing/it/test_whitelist.py index d687da7cb..a380062ad 100644 --- a/tests/nemo_text_processing/it/test_whitelist.py +++ b/tests/nemo_text_processing/it/test_whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/ja/__init__.py b/tests/nemo_text_processing/ja/__init__.py new file mode 100644 index 000000000..4fc50543f --- /dev/null +++ b/tests/nemo_text_processing/ja/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..c763a3d87 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,28 @@ +一~1 +百~100 +五千億~500,000,000,000 +五万~50,000 +五兆~5,000,000,000,000 +十一兆~11,000,000,000,000 +十一兆一~11,000,000,000,001 +九十九兆~99,000,000,000,000 +一兆~1,000,000,000,000 +一兆一~1,000,000,000,001 +一兆十~1,000,000,000,010 +一兆百~1,000,000,000,100 +一兆千~1,000,000,001,000 +一兆一万~1,000,000,010,000 +一兆十万~1,000,000,100,000 +一兆百万~1,000,001,000,000 +一兆一千万~1,000,010,000,000 +そこに鳥一羽がいます~そこに鳥1羽がいます +これから百数えてください~これから100数えてください +生産に掛かる費用は五千億になります~生産に掛かる費用は500,000,000,000になります +お年玉五万あげる~お年玉50,000あげる +五兆円分の株式を買った~5,000,000,000,000円分の株式を買った +今年の収益は十一兆になる~今年の収益は11,000,000,000,000になる +隣の会社の年収益は九十九兆だそうだ~隣の会社の年収益は99,000,000,000,000だそうだ +政府は一兆の赤字で困っている~政府は1,000,000,000,000の赤字で困っている +兵士五百人を派遣する~兵士500人を派遣する +お寺に一万寄付した~お寺に10,000寄付した +クラスに二十人いる~クラスに20人いる \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..c22d26310 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,31 @@ +一日~1日 +一月~1月 +一月一日~1月1日 +一月二十二日~1月22日 +七十から八十年代~70〜80年代 +七十年代~70年代 +七月~7月 +七月二十三日~7月23日 +八月四日~8月4日 +五から九日~5〜9日 +九月~9月 +三から四月~3〜4月 +三月一日水曜日~3月1日(水) +四月三十日日曜日~4月30日(日) +三月二十日~3月20日 +三月~3月 +九十年代~90年代 +九月~9月 +九月五日~9月5日 +二十一世紀~21世紀 +二十一日月曜日~21日(月) +今日は一月二十二日~今日は1月22日 +毎月の三十日はゴミの日~毎月の30日はゴミの日 +七十年代はロックがはやってた~70年代はロックがはやってた +二十一世紀でやることじゃない~21世紀でやることじゃない +正月は一月一日から始まる~正月は1月1日から始まる +五から九日は休みの日~5〜9日は休みの日 +誕生日は千九百九十九年三月二十日~誕生日は1999年3月20日 +八月と九月はまだ夏~8月と9月はまだ夏 +放送日は三月一日水曜日~放送日は3月1日(水) +十月になるとすずしい~10月になるとすずしい diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..d5080acd0 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,32 @@ +マイナス一点零六~-1.06 +マイナス七点零零六~-7.006 +マイナス三十九点五七四~-39.574 +マイナス三点八六~-3.86 +マイナス九十二点一五七四~-92.1574 +マイナス九点零三八~-9.038 +マイナス二点八七四一~-2.8741 +マイナス二百三十一点四六零九~-231.4609 +マイナス五十二点一八~-52.18 +マイナス五点三~-5.3 +マイナス五百七十九点三零零二~-579.3002 +マイナス八十六点四~-86.4 +マイナス八点四零九~-8.409 +マイナス八百二十一点七九五四~-821.7954 +マイナス八百五十二点七~-852.7 +マイナス六十一点零七~-61.07 +マイナス六点八一四~-6.814 +マイナス六百五十七点三零二四~-657.3024 +マイナス四十二点六零五~-42.605 +マイナス四百八十九点零五二一~-489.0521 +答えはマイナス一点零六~答えは-1.06 +計算の結果はマイナス七点零零六~計算の結果は-7.006 +マイナス二点八七四はかなり悪いスコア~-2.874はかなり悪いスコア +五点三は平均点~5.3は平均点 +テストの点数は八十六点四~テストの点数は86.4 +マイナス三十九点五七四は低すぎる~-39.574は低すぎる +答えはマイナス一点零六~答えは-1.06 +計算の結果はマイナス八十六点四~計算の結果は-86.4 +マイナス五十二点一八はかなり悪いスコア~-52.18はかなり悪いスコア +六点八一四は平均点~6.814は平均点 +テストの点数は九十二点一五七四~テストの点数は92.1574 +マイナス七点零零六は低すぎる~-7.006は低すぎる diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..32f80e812 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt @@ -0,0 +1,34 @@ +マイナス一と四分の三~-1 3/4 +一と四分の三~1 3/4 +マイナス一分の九~-9/1 +マイナス一分の六十~-60/1 +マイナス一分の百二十三~-123/1 +マイナス一荷四分の三~-1 3/4 +マイナス七百二十分の一~-1/720 +マイナス三十二分の三十一~-31/32 +マイナス三百九十七分の四~-4/397 +マイナス三百五十分の一~-1/350 +マイナス九十八分の四百七十一~-471/98 +マイナス二と五分の三~-2 3/5 +マイナス二十分の九~-9/20 +マイナス二十分の二十一~-21/20 +マイナス二十四分の一~-1/24 +マイナス二百二十分の一~-1/220 +マイナス二百五十二分の百四十七~-147/252 +マイナス二百五十六分の一~-1/256 +マイナス二荷五分の三~-2 3/5 +マイナス五分の七~-7/5 +マイナス五分の八~-8/5 +マイナス五分の十四~-14/5 +マイナス五分の百三十二~-132/5 +マイナス八分の五~-5/8 +答えはマイナス八分の五~答えは-5/8 +三分の一の人がその場を離れた~1/3の人がその場を離れた +約二分の一を削る~約1/2を削る +十分の三を削って吟醸をつくる~3/10を削って吟醸をつくる +一人三分の一ぐらい取る~1人1/3ぐらい取る +答えは九分の一~答えは1/9 +三分の二の人がその場を離れた~2/3の人がその場を離れた +約十分の一を削る~約1/10を削る +三分の一を削って吟醸をつくる~1/3を削って吟醸をつくる +一人二分の一とぐらい取る~1人1/2とぐらい取る \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..873f3f9dd --- /dev/null +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,65 @@ +一万一番目~10001番目 +一万番目~10000番目 +一番目~1番目 +七十番目~70番目 +七千番目~7000番目 +七番目~7番目 +七百番目~700番目 +三十番目~30番目 +三千三百三十番目~3330番目 +三千番目~3000番目 +三番目~3番目 +三百番目~300番目 +九十番目~90番目 +九千九百九十九番目~9999番目 +九千番目~9000番目 +九番目~9番目 +九百番目~900番目 +二十番目~20番目 +二千二百番目~2200番目 +二千番目~2000番目 +二番目~2番目 +二百番目~200番目 +五十番目~50番目 +五千番目~5000番目 +五番目~5番目 +五百番目~500番目 +八十番目~80番目 +八千番目~8000番目 +第一~第1 +第一万~第10000 +第一万一~第10001 +第一万九千~第19000 +第一万九千八百~第19800 +第七~第7 +第七万~第70000 +第七万二千六~第72006 +第七十~第70 +第七十二~第72 +第七千~第7000 +第七千八十九~第7089 +第七百~第700 +第七百三十~第730 +第七百三十五~第735 +第七百九~第709 +第三~第3 +第三万四~第30004 +第三十~第30 +第三千~第3000 +第三千三百二十二~第3322 +第三千十七~第3017 +第三千四百一~第3401 +第三百~第300 +第九~第9 +第九万~第90000 +第九十~第90 +五番目私の席~5番目私の席 +第七班に任務を任せる~第7班に任務を任せる +この角からまっすぐ行って三番目の交差点で曲がる~この角からまっすぐ行って3番目の交差点で曲がる +田中君は二番目の席~田中君は2番目の席 +トップから数えて第七十二~トップから数えて第72 +八番目私の席~8番目私の席 +第十三班に任務を任せる~第13班に任務を任せる +この角からまっすぐ行って二番目の交差点で曲がる~この角からまっすぐ行って2番目の交差点で曲がる +田中君は五番目の席~田中君は5番目の席 +トップから数えて第89~トップから数えて第89 diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..6a5082124 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,40 @@ +七時一分~7時1分 +七時四分~7時4分 +九時五十八分~9時58分 +九時十分前~9時10分前 +九時四十分~9時40分 +五時二十六分~5時26分 +六時五十五分~6時55分 +三時~3時 +三時~3時 +正午一分前~正午1分前 +正午十分過ぎ~正午10分過ぎ +九時三十分~9時30分 +七時五十分頃~7時50分頃 +一時~1時 +一時十分~1時10分 +三時~3時 +十七時~17時 +二十時~20時 +二十一時~21時 +二時~2時 +十二時三十分~12時30分 +零時~0時 +零時一分前~0時1分前 +二時~2時 +十二時~12時 +二十時~20時 +二十三時~23時 +二十四時~24時 +零時~0時 +四時~4時 +毎日五時に起きる~毎日5時に起きる +九時四十分の予約になります~9時40分の予約になります +現在の時間は十二時三十分~現在の時間は12時30分 +ちょうど零時になった~ちょうど0時になった +四時で店を閉める~4時で店を閉める +毎日六時に起きる~毎日6時に起きる +十時三十分の予約になります~10時30分の予約になります +現在の時間は十時三分~現在の時間は10時3分 +ちょうど一時になった~ちょうど1時になった +七時で店を閉める~7時で店を閉める diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..1d8a2801a --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,43 @@ +1~一 +2~二 +10~十 +100~百 +1000~千 +5000~五千 +10000~一万 +120000~十二万 +1230000~百二十三万 +12345000~一千二百三十四万五千 +123456700~一億二千三百四十五万六千七百 +1234567890~十二億三千四百五十六万七千八百九十 +12343212400~百二十三億四千三百二十一万二千四百 +101~百一 +1002~千二 +10003~一万三 +100004~十万四 +1000005~百万五 +10000006~一千万六 +100000008~一億八 +1000000009~十億九 +10000000009~百億九 +100000000009~一千億九 +100000000090~一千億九十 +100000000900~一千億九百 +100000009000~一千億九千 +100000090000~一千億九万 +100000900000~一千億九十万 +100009000000~一千億九百万 +100090000000~一千億九千万 +100900000000~一千九億 +109000000000~一千九十億 +そこに鳥1羽がいます~そこに鳥一羽がいます +これから100数えてください~これから百数えてください +生産に掛かる費用は500000000000になります~生産に掛かる費用は五千億になります +お年玉50000あげる~お年玉五万あげる +500000000000円分の株式を買った~五千億円分の株式を買った +今年の収益は100000000000になる~今年の収益は一千億になる +隣の会社の年収益は990000000000だそうだ~隣の会社の年収益は九千九百億だそうだ +政府は100000000000の赤字で困っている~政府は一千億の赤字で困っている +兵士500人を派遣する~兵士五百人を派遣する +お寺に10000寄付した~お寺に一万寄付した +クラスに20人いる~クラスに二十人いる diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..6625ad4d0 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt @@ -0,0 +1,200 @@ +2024/01/30~二千二十四年一月三十日 +2024年~二千二十四年 +R.4~令和四年 +3月~三月 +15日~十五日 +1月1日(月)〜3日(水)~一月一日月曜日から三日水曜日 +1日から来年2月末まで~一日から来年二月末まで +1月〜12月~一月から十二月 +1月の最終金曜日~一月の最終金曜日 +1月1日(月)〜3日(水)~一月一日月曜日から三日水曜日 +1月22日~一月二十二日 +70〜80年代~七十から八十年代 +70年代~七十年代 +7月中~七月中 +7月23日と8月4日~七月二十三日と八月四日 +7月5〜9日(月〜金)~七月五から九日月曜日から金曜日 +7月初旬〜9月中旬~七月初旬から九月中旬 +3〜4月~三から四月 +3月から7月まで~三月から七月まで +3月1日(水)から4月30日(日)~三月一日水曜日から四月三十日日曜日 +3月20日〜25日~三月二十日から二十五日 +3月半ば~三月半ば +90年代~九十年代 +9月中旬~九月中旬 +9月5日から9月10日まで~九月五日から九月十日まで +21世紀~二十一世紀 +21日月曜日~二十一日月曜日 +23年上半期~二十三年上半期 +23年2月25日(土)~二十三年二月二十五日土曜日 +20世紀〜21世紀~二十世紀から二十一世紀 +26日夜~二十六日夜 +24日水曜日~二十四日水曜日 +2003年6月19日~二千三年六月十九日 +2009年3〜4月~二千九年三から四月 +2021年7月〜8月~二千二十一年七月から八月 +2021年7月1〜7月3日~二千二十一年七月一から七月三日 +2021年12月末~二千二十一年十二月末 +2021年4月~二千二十一年四月 +2027年7月~二千二十七年七月 +2023年1月2日(月・祝)~二千二十三年一月二日月曜日祝日 +2023年3月1日(水)〜6月12日(火)~二千二十三年三月一日水曜日から六月十二日火曜日 +2023年2月~二千二十三年二月 +2022FIFAワールドカップ~二千二十二FIFAワールドカップ +24年に再開~二十四年に再開 +2025年12月13日~二千二十五年十二月十三日 +2024年12月26日(木)~二千二十四年十二月二十六日木曜日 +25年関西万博~二十五年関西万博 +2020年3月~二千二十年三月 +2020〜2030年予測~二千二十から二千三十年予測 +2019年5月~二千十九年五月 +2012年8月8日(水)~二千十二年八月八日水曜日 +2015年1月~二千十五年一月 +2015年10月から2021年12月~二千十五年十月から二千二十一年十二月 +2018年1月22〜27日~二千十八年一月二十二から二十七日 +2018年3月8日~二千十八年三月八日 +2010年代~二千十年代 +2004年11月7日から2005年4月14日~二千四年十一月七日から二千五年四月十四日 +2000年代~二千年代 +2000年4月4日(火)~二千年四月四日火曜日 +2月21日〜3月7日~二月二十一日から三月七日 +2月4日(土)、5日(日)~二月四日土曜日、五日日曜日 +5月~五月 +5月30日(日)~五月三十日日曜日 +5月、9月、12月~五月、九月、十二月 +今年3月3日~今年三月三日 +今年10月~今年十月 +今月20日、26日、27日~今月二十日、二十六日、二十七日 +今月11日(木)と22日(日)~今月十一日木曜日と二十二日日曜日 +今週5月30日〜6月3日~今週五月三十日から六月三日 +今週末20日~今週末二十日 +令和3年6月~令和三年六月 +令和2〜4年~令和二から四年 +令和2年7月25日~令和二年七月二十五日 +令和元年10月から令和2年3月~令和元年十月から令和二年三月 +令和4年~令和四年 +令和4年9月25日(日)~令和四年九月二十五日日曜日 +令和4年12月~令和四年十二月 +令和4年5月1日(土)から6月6日(金)~令和四年五月一日土曜日から六月六日金曜日 +元禄14年~元禄十四年 +先々週12日~先々週十二日 +先月25〜27日~先月二十五から二十七日 +先月4日~先月四日 +80年代~八十年代 +8月と12月~八月と十二月 +8月21日~八月二十一日 +8月末日~八月末日 +60年代後期〜70年代初期~六十年代後期から七十年代初期 +6月下旬~六月下旬 +6月17日(土)から20日(火)まで~六月十七日土曜日から二十日火曜日まで +6月18日(土)~六月十八日土曜日 +再放送は5日~再放送は五日 +再来週20日~再来週二十日 +11月~十一月 +11月1日〜11月10日~十一月一日から十一月十日 +11月5日から12月9日まで~十一月五日から十二月九日まで +11月18〜27日~十一月十八から二十七日 +11月16日(水)~十一月十六日水曜日 +17世紀〜18世紀中頃~十七世紀から十八世紀中頃 +13〜14世紀~十三から十四世紀 +19世紀~十九世紀 +12日未明~十二日未明 +12月~十二月 +12月から来年3月まで~十二月から来年三月まで +12月11日~十二月十一日 +12月19日〜23日(火〜土)~十二月十九日から二十三日火曜日から土曜日 +16世紀〜20世紀初頭~十六世紀から二十世紀初頭 +16世紀から1945年まで~十六世紀から千九百四十五年まで +10日〜15日の期間限定~十日から十五日の期間限定 +10月中旬〜11月上旬~十月中旬から十一月上旬 +10月27日(金)〜11月9日(木)~十月二十七日金曜日から十一月九日木曜日 +10月28日大安~十月二十八日大安 +10月末~十月末 +1700年代初頭~千七百年代初頭 +1977年11月15日~千九百七十七年十一月十五日 +1972年5月5日(金)~千九百七十二年五月五日金曜日 +1976年7月17日〜8月1日~千九百七十六年七月十七日から八月一日 +1998年2月2日~千九百九十八年二月二日 +1994年9月22日〜24日~千九百九十四年九月二十二日から二十四日 +1990年代~千九百九十年代 +1990年11月17日(土)~千九百九十年十一月十七日土曜日 +1985年5月19日~千九百八十五年五月十九日 +1980年代後半〜90年代前半~千九百八十年代後半から九十年代前半 +1961年10月11日~千九百六十一年十月十一日 +1966年8月24日~千九百六十六年八月二十四日 +1964年~千九百六十四年 +1960年9月~千九百六十年九月 +1940年代〜1990年代~千九百四十年代から千九百九十年代 +1200年代末期から1500年代~千二百年代末期から千五百年代 +1870年代~千八百七十年代 +1868年から1912年までの44年間~千八百六十八年から千九百十二年までの四十四年間 +1180年代頃~千百八十年代頃 +4月1日~四月一日 +4月上旬~四月上旬 +4月29日〜5月10日~四月二十九日から五月十日 +大正15年~大正十五年 +寛永6年~寛永六年 +平成31年4月から令和2年3月~平成三十一年四月から令和二年三月 +平成30年~平成三十年 +平成30年2月3日(土)〜2月8日(木)~平成三十年二月三日土曜日から二月八日木曜日 +平成30年8月~平成三十年八月 +平成30年6月14日(木)~平成三十年六月十四日木曜日 +平成20〜30年代~平成二十から三十年代 +平成26年7月30日〜8月11日~平成二十六年七月三十日から八月十一日 +平成20年代~平成二十年代 +平成5年から令和元年まで~平成五年から令和元年まで +平成元年〜9年~平成元年から九年 +平成元年1月2日~平成元年一月二日 +平成17年12月〜平成18年1月~平成十七年十二月から平成十八年一月 +平成16年4月2日から令和5年2月28日~平成十六年四月二日から令和五年二月二十八日 +平成10年10月10日~平成十年十月十日 +旧暦2月~旧暦二月 +明後日2月20日~明後日二月二十日 +明日から8月末日~明日から八月末日 +明日5日~明日五日 +昨年11月~昨年十一月 +昭和38年~昭和三十八年 +昭和30年代~昭和三十年代 +昭和55年~昭和五十五年 +昭和60〜64年~昭和六十から六十四年 +昭和62年~昭和六十二年 +昭和15年1月~昭和十五年一月 +昭和43年4月30日~昭和四十三年四月三十日 +昭和40年代から60年代の間~昭和四十年代から六十年代の間 +来年の9月15日~来年の九月十五日 +来年4月~来年四月 +来週10月10日〜10月14日~来週十月十日から十月十四日 +来週金曜日3日~来週金曜日三日 +毎年1月2日から3日~毎年一月二日から三日 +毎月10日~毎月十日 +紀元33年~紀元三十三年 +紀元前7世紀から西暦20世紀まで~紀元前七世紀から西暦二十世紀まで +紀元前721年~紀元前七百二十一年 +紀元前3000年〜300年頃~紀元前三千年から三百年頃 +紀元前900〜500年代~紀元前九百から五百年代 +紀元前2世紀〜西暦1200年頃~紀元前二世紀から西暦千二百年頃 +紀元前2500年代〜紀元前1800年代~紀元前二千五百年代から紀元前千八百年代 +紀元前626年〜紀元75年~紀元前六百二十六年から紀元七十五年 +紀元前660年2月11日~紀元前六百六十年二月十一日 +紀元前10〜8世紀~紀元前十から八世紀 +紀元前12〜同9世紀~紀元前十二から同九世紀 +紀元前15世紀~紀元前十五世紀 +紀元前1500年頃~紀元前千五百年頃 +紀元前1600年代~紀元前千六百年代 +紀元前4000年から紀元前6世紀まで~紀元前四千年から紀元前六世紀まで +紀元前100万年~紀元前百万年 +紀元前108年~紀元前百八年 +紀元後1世紀~紀元後一世紀 +紀元後6年~紀元後六年 +紀元後4〜7年~紀元後四から七年 +紀元後40年〜220年頃~紀元後四十年から二百二十年頃 +紀元後170年代前半~紀元後百七十年代前半 +締切は9日~締切は九日 +翌月1日~翌月一日 +西暦794年~西暦七百九十四年 +西暦3〜6世紀~西暦三から六世紀 +西暦3世紀~西暦三世紀 +西暦1203〜1204年~西暦千二百三から千二百四年 +西暦1050年~西暦千五十年 +西暦1868年10月23日~西暦千八百六十八年十月二十三日 +来年は西暦1868年10月23日ですか~来年は西暦千八百六十八年十月二十三日ですか diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..622b61d1a --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,15 @@ +この本は2.5センチの厚さです。~この本は二点五センチの厚さです。 +今日の気温は25.3度です。~今日の気温は二十五点三度です。 +私の車は1リットルで15.8キロ走ります。~私の車は一リットルで十五点八キロ走ります。 +彼の体重は72.4キログラムです。~彼の体重は七十二点四キログラムです。 +昨日、2.3キロの魚を釣りました。~昨日、二点三キロの魚を釣りました。 +この道の幅は3.6メートルです。~この道の幅は三点六メートルです。 +コーヒー1杯には0.5グラムのカフェインが含まれています。~コーヒー一杯には零点五グラムのカフェインが含まれています。 +その映画は二時間三十分、つまり2.5時間続きました。~その映画は二時間三十分、つまり二点五時間続きました。 +彼女の身長は165.4センチです。~彼女の身長は百六十五点四センチです。 +昨夜の最低気温は-2.5度でした。~昨夜の最低気温はマイナス二点五度でした。 +その山の斜面は-30.8度の傾斜があります。~その山の斜面はマイナス三十点八度の傾斜があります。 +今年の冬の平均気温は-4.2度でした。~今年の冬の平均気温はマイナス四点二度でした。 +昨日の株価は-3.6ポイント下落しました。~昨日の株価はマイナス三点六ポイント下落しました。 +彼の体重は-2.4キログラム減りました。~彼の体重はマイナス二点四キログラム減りました。 +その製品の評価は-1.5ポイントでした。~その製品の評価はマイナス一点五ポイントでした。 \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..e2095fbfa --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,18 @@ +1/2~二分の一 +-1/2~マイナス二分の一 +1 1/2~一荷二分の一 +1と1/2~一と二分の一 +1荷1/2~一荷二分の一 +-1荷1/2~マイナス一荷二分の一 +マイナス1荷1/2~マイナス一荷二分の一 +マイナス√1荷1/2~マイナスルート一荷二分の一 +-√1荷1/2~マイナスルート一荷二分の一 +-1荷√1/2~マイナス一荷二分のルート一 +3分の1~三分の一 +-3分の1~マイナス三分の一 +-√3分の1~マイナスルート三分の一 +1荷√1/2~一荷二分のルート一 +1荷√1/3~一荷三分のルート一 +1と1/3~一と三分の一 +1荷√1/4~一荷四分のルート一 +1と√1/4~一と四分のルート一 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..597361aea --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,64 @@ +第1~第一 +第2~第二 +第10~第十 +第100~第百 +第1000~第千 +第5000~第五千 +第10000~第一万 +第120000~第十二万 +第1230000~第百二十三万 +第12345000~第一千二百三十四万五千 +第123456700~第一億二千三百四十五万六千七百 +第1234567890~第十二億三千四百五十六万七千八百九十 +第12343212400~第百二十三億四千三百二十一万二千四百 +第101~第百一 +第1002~第千二 +第10003~第一万三 +第100004~第十万四 +第1000005~第百万五 +第10000006~第一千万六 +第100000008~第一億八 +第1000000009~第十億九 +第10000000009~第百億九 +第100000000009~第一千億九 +第100000000090~第一千億九十 +第100000000900~第一千億九百 +第100000009000~第一千億九千 +第100000090000~第一千億九万 +第100000900000~第一千億九十万 +第100009000000~第一千億九百万 +第100090000000~第一千億九千万 +第100900000000~第一千九億 +第109000000000~第一千九十億 +1~一 +2~二 +10~十 +100~百 +1000~千 +5000~五千 +10000~一万 +120000~十二万 +1230000~百二十三万 +12345000~一千二百三十四万五千 +123456700~一億二千三百四十五万六千七百 +1234567890~十二億三千四百五十六万七千八百九十 +12343212400~百二十三億四千三百二十一万二千四百 +101番目~百一番目 +1002番目~千二番目 +10003番目~一万三番目 +100004番目~十万四番目 +1000005番目~百万五番目 +10000006番目~一千万六番目 +100000008番目~一億八番目 +1000000009番目~十億九番目 +10000000009番目~百億九番目 +100000000009番目~一千億九番目 +100000000090番目~一千億九十番目 +100000000900番目~一千億九百番目 +100000009000番目~一千億九千番目 +100000090000番目~一千億九万番目 +100000900000番目~一千億九十万番目 +100009000000番目~一千億九百万番目 +100090000000番目~一千億九千万番目 +100900000000番目~一千九億番目 +109000000000番目~一千九十億番目 \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..42e7600a7 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt @@ -0,0 +1,202 @@ +1時30分~一時三十分 +1時13分~一時十三分 +1時19分~一時十九分 +7時3分~七時三分 +7時28分~七時二十八分 +3時21分~三時二十一分 +3時15分前~三時十五分前 +3時18分~三時十八分 +3時10分~三時十分 +9時31分~九時三十一分 +9時9分~九時九分 +9時15分過ぎ~九時十五分過ぎ +9時半~九時半 +21時3分~二十一時三分 +21時10分前~二十一時十分前 +21時49分~二十一時四十九分 +21時45分~二十一時四十五分 +27時~二十七時 +23時37分~二十三時三十七分 +23時59分~二十三時五十九分 +23時44分~二十三時四十四分 +22時51分~二十二時五十一分 +22時16分~二十二時十六分 +22時41分~二十二時四十一分 +25時~二十五時 +25時20分~二十五時二十分 +26時~二十六時 +24時~二十四時 +24時25分~二十四時二十五分 +24時15分~二十四時十五分 +24時半~二十四時半 +20時52分~二十時五十二分 +20時15分~二十時十五分 +20時43分~二十時四十三分 +2時~二時 +2時57分~二時五十七分 +2時54分~二時五十四分 +2時12分~二時十二分 +2時頃~二時頃 +5時33分~五時三十三分 +5時2分~五時二分 +5時20分過ぎ~五時二十分過ぎ +今夜0時~今夜零時 +今晩7時20分~今晩七時二十分 +今朝6時頃~今朝六時頃 +8時33分~八時三十三分 +8時8分~八時八分 +8時10分過ぎ~八時十分過ぎ +8時半頃~八時半頃 +6時39分~六時三十九分 +6時24分~六時二十四分 +11時~十一時 +11時30分~十一時三十分 +11時25分~十一時二十五分 +11時55分~十一時五十五分 +17時23分~十七時二十三分 +17時53分~十七時五十三分 +17時8分~十七時八分 +13時3分~十三時三分 +13時35分~十三時三十五分 +13時27分~十三時二十七分 +19時36分~十九時三十六分 +19時48分~十九時四十八分 +12時1分~十二時一分 +12時20分前~十二時二十分前 +12時10分~十二時十分 +15時35分~十五時三十五分 +15時15分前~十五時十五分前 +15時14分~十五時十四分 +15時台~十五時台 +18時26分~十八時二十六分 +18時5分前~十八時五分前 +18時50分~十八時五十分 +16時38分~十六時三十八分 +16時20分~十六時二十分 +16時台~十六時台 +14時7分~十四時七分 +14時59分~十四時五十九分 +14時4分~十四時四分 +10時から18時~十時から十八時 +10時29分~十時二十九分 +10時25分から15時55分~十時二十五分から十五時五十五分 +10時5分前~十時五分前 +10時47分~十時四十七分 +午前1時~午前一時 +午前1時29分~午前一時二十九分 +午前1時53分~午前一時五十三分 +午前1時50分~午前一時五十分 +午前1時頃~午前一時頃 +午前7時30分~午前七時三十分 +午前3時~午前三時 +午前3時11分~午前三時十一分 +午前9から12時まで~午前九から十二時まで +午前9時~午前九時 +午前9時34分~午前九時三十四分 +午前2時~午前二時 +午前2時51分~午前二時五十一分 +午前5時15分~午前五時十五分 +午前8時5分過ぎ~午前八時五分過ぎ +午前8時6分~午前八時六分 +午前8時15分~午前八時十五分 +午前6時~午前六時 +午前6時2分~午前六時二分 +午前12時10分~午前十二時十分 +午前4時~午前四時 +午前4時40分~午前四時四十分 +午前0時~午前零時 +午前0時14分~午前零時十四分 +午前0時台~午前零時台 +午前0時45分~午前零時四十五分 +午後1時~午後一時 +午後1時30分~午後一時三十分 +午後1時9分~午後一時九分 +午後1時23分~午後一時二十三分 +午後1時20分~午後一時二十分 +午後1時59分~午後一時五十九分 +午後3時25分~午後三時二十五分 +午後3時20分前~午後三時二十分前 +午後3時5分~午後三時五分 +午後3時40分過ぎ~午後三時四十分過ぎ +午後3時頃~午後三時頃 +午後9時3分~午後九時三分 +午後9時17分~午後九時十七分 +午後2時~午後二時 +午後2時6分~午後二時六分 +午後2時19分~午後二時十九分 +午後2時49分~午後二時四十九分 +午後2時46分~午後二時四十六分 +午後2時過ぎ~午後二時過ぎ +午後8時台~午後八時台 +午後6時~午後六時 +午後6時22分~午後六時二十二分 +午後12時49分~午後十二時四十九分 +午後4時11分~午後四時十一分 +午後4時10分~午後四時十分 +午後4時42分~午後四時四十二分 +午後0時~午後零時 +午後0時1分~午後零時一分 +午後0時30分~午後零時三十分 +午後0時15分~午後零時十五分 +午後0時半~午後零時半 +4時32分~四時三十二分 +4時56分~四時五十六分 +4時43分~四時四十三分 +夕方5時~夕方五時 +夕方5時半~夕方五時半 +夕方5時台~夕方五時台 +夕方5時頃~夕方五時頃 +夕方4時15分~夕方四時十五分 +夜の7時~夜の七時 +夜の12時45分~夜の十二時四十五分 +夜1時~夜一時 +夜中の3時1分~夜中の三時一分 +夜中の2時~夜中の二時 +夜中の2時50分~夜中の二時五十分 +夜中2時~夜中二時 +夜8時4分~夜八時四分 +夜8時45分~夜八時四十五分 +夜半11時~夜半十一時 +夜0時5分~夜零時五分 +日本時間午前2時~日本時間午前二時 +日本時間午後9時~日本時間午後九時 +日本時間午後9時35分~日本時間午後九時三十五分 +日本時間夜中2時5分~日本時間夜中二時五分 +早朝5時半~早朝五時半 +早朝4時~早朝四時 +明け方5時~明け方五時 +明け方4時半~明け方四時半 +昼の1時過ぎ~昼の一時過ぎ +昼の2時~昼の二時 +昼の2時30分~昼の二時三十分 +昼12時~昼十二時 +朝の7時~朝の七時 +朝7時1分~朝七時一分 +朝7時4分~朝七時四分 +朝9時58分~朝九時五十八分 +朝9時10分前~朝九時十分前 +朝9時40分~朝九時四十分 +朝5時26分~朝五時二十六分 +朝6時55分~朝六時五十五分 +朝方3時~朝方三時 +未明3時~未明三時 +正午1分前~正午一分前 +正午10分過ぎ~正午十分過ぎ +毎夜9時30分~毎夜九時三十分 +毎朝7時50分頃~毎朝七時五十分頃 +深夜1時~深夜一時 +深夜1時10分~深夜一時十分 +深夜3時~深夜三時 +深夜27時~深夜二十七時 +深夜25時~深夜二十五時 +深夜26時~深夜二十六時 +深夜2時~深夜二時 +深夜12時30分~深夜十二時三十分 +深夜0時~深夜零時 +深夜0時1分前~深夜零時一分前 +真夜中の2時~真夜中の二時 +真夜中の12時~真夜中の十二時 +翌日27時~翌日二十七時 +翌日25時~翌日二十五時 +翌日26時~翌日二十六時 +0時~零時 \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/test_cardinal.py b/tests/nemo_text_processing/ja/test_cardinal.py new file mode 100644 index 000000000..0c8faf728 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_cardinal.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_date(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds + + inverse_normalizer_ja = InverseNormalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ja/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_ja.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ja/test_date.py b/tests/nemo_text_processing/ja/test_date.py new file mode 100644 index 000000000..cd7127698 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_date.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + + inverse_normalizer = InverseNormalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ja/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ja/test_decimal.py b/tests/nemo_text_processing/ja/test_decimal.py new file mode 100644 index 000000000..069f9fb3f --- /dev/null +++ b/tests/nemo_text_processing/ja/test_decimal.py @@ -0,0 +1,41 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_date(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ja/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ja/test_fraction.py b/tests/nemo_text_processing/ja/test_fraction.py new file mode 100644 index 000000000..98d49229b --- /dev/null +++ b/tests/nemo_text_processing/ja/test_fraction.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFraction: + + inverse_normalizer = InverseNormalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ja/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ja/test_ordinal.py b/tests/nemo_text_processing/ja/test_ordinal.py new file mode 100644 index 000000000..ffd9ea511 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_ordinal.py @@ -0,0 +1,41 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_date(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ja/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ja/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/ja/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..bef2a9666 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,55 @@ +#! /bin/sh + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration_pp.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} +testITNFractionText() { + input=$TEST_DIR/ja/data_inverse_text_normalization/test_cases_fraction.txt + runtest $input +} +testITNCardinalText() { + input=$TEST_DIR/ja/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} +testITNOrdinalText() { + input=$TEST_DIR/ja/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} +testITNDateText() { + input=$TEST_DIR/ja/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} +testITNDecimalText() { + input=$TEST_DIR/ja/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} +testITNTimeText() { + input=$TEST_DIR/ja/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + + +# Remove all command-line arguments +shift $# + +# Load shUnit2 +#. $PROJECT_DIR/../shunit2/shunit2 +. /workspace/shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..42db11fd6 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh @@ -0,0 +1,53 @@ +#! /bin/sh + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read written spoken <<< $testcase + # replace non breaking space with breaking space + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration_pp.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} +testTNFractionText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_fraction.txt + runtest $input +} +testTNTimeText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_time.txt + runtest $input +} +testTNCardinalText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_cardinal.txt + runtest $input +} +testTNOrdinalText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_ordinal.txt + runtest $input +} +testTNDecimalalText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_decimal.txt + runtest $input +} +testTNDateText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_date.txt + runtest $input +} + + +# Load shUnit2 +#. $PROJECT_DIR/../shunit2/shunit2 +. /workspace/shunit2/shunit2 diff --git a/tests/nemo_text_processing/ja/test_time.py b/tests/nemo_text_processing/ja/test_time.py new file mode 100644 index 000000000..6d81e3e5b --- /dev/null +++ b/tests/nemo_text_processing/ja/test_time.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + + inverse_normalizer = InverseNormalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ja/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + + normalizer = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/__init__.py b/tests/nemo_text_processing/ko/__init__.py new file mode 100644 index 000000000..341a77c5b --- /dev/null +++ b/tests/nemo_text_processing/ko/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..50c58425a --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,37 @@ +영~0 +구~9 +십~10 +십칠~17 +오십삼~53 +백~100 +백오~105 +삼백이십~320 +구백팔십칠~987 +천~1000 +천육~1006 +천오백~1500 +오천사백삼십이~5432 +만~10000 +만천이백~11200 +삼만오천칠백~35700 +십이만~120000 +백오십만삼천~1503000 +천만~10000000 +오천이백칠십만육천백~52706100 +억~100000000 +삼억오천만~350000000 +십이억천만~1210000000 +백오십억칠천만~15070000000 +오천억~500000000000 +일조~1000000000000 +이조오천억~2500000000000 +영영영~0 0 0 +영영백이십삼~0 0 123 +만천~11000 +만천백십일~11111 +경~10000000000000000 +마이너스일~-1 +마이너스 일~-1 +- 일~-1 +마이너스일억사천이백칠십구만구천팔십이~-142799082 +마이너스 칠백삼십오~-735 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..89fa75eb6 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,15 @@ +이천이십사년팔월이십팔일~2024년8월28일 +이천이십삼년 구월 오일~2023년9월5일 +천구백구십구년십이월삼십일일~1999년12월31일 +이천년 이월 이십구일~2000년2월29일 +이천십년시월십일~2010년10월10일 +이천이십일년유월십육일~2021년6월16일 +이천삼십년삼월십사일~2030년3월14일 +천구백팔십팔년 오월 이십일~1988년5월20일 +이천일년 칠월 구일~2001년7월9일 +이천십팔년사월삼십일~2018년4월30일 +삼천년팔월십오일~3000년8월15일 +이천구년 일월이십일~2009년1월20일 +이천삼십오년~2035년 +오월~5월 +구천구백구십구년삼월일일~9999년3월1일 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..63d023168 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,14 @@ +일점삼~1.3 +영점오~0.5 +십점오~10.5 +이십삼점사~23.4 +백점일~100.1 +일점이삼~1.23 +영점오육칠~0.567 +구십구점구구~99.99 +일점영삼~1.03 +영점영영일~0.001 +천이백삼십사점오육~1234.56 +일점오만~1.5만 +일점오억~1.5억 +일점오경~1.5경 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..c5fda707d --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_fraction.txt @@ -0,0 +1,29 @@ +이분의일~1/2 +사분의일~1/4 +사분의삼~3/4 +오분의이~2/5 +십분의칠~7/10 +십이분의오~5/12 +이십삼분의십~10/23 +백분의일~1/100 +백분의구십구~99/100 +천분의백이십삼~123/1000 +일과이분의일~1 1/2 +삼과사분의일~3 1/4 +오와팔분의삼~5 3/8 +십과백분의칠십오~10 75/100 +마이너스사분의일~-1/4 +영점오분의일~1/0.5 +삼분의일점오~1.5/3 +루트사분의일~1/√4 +구분의루트십육~√16/9 +이와루트구분의일~2 1/√9 +마이너스오분의루트이십오~-√25/5 +칠분의육~6/7 +백오십분의이십~20/150 +사와오분의이~4 2/5 +이십과백분의일~20 1/100 +일점오분의영점이~0.2/1.5 +루트백분의십~10/√100 +십과루트팔십일분의삼~10 3/√81 +마이너스이와십분의일~-2 1/10 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..fa020a49c --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_measure.txt @@ -0,0 +1,46 @@ +오 킬로미터~5 km +백 미터~100 m +이십 센티미터~20 cm +삼십 밀리미터~30 mm +십이 점 일구오 킬로미터~12.195 km +천오백 미터~1500 m +영 점 오 밀리미터~0.5 mm +칠 미터~7 m +오십 킬로그램~50 kg +이백 그램~200 g +삼 톤~3 t +일 점 오 킬로그램~1.5 kg +천 밀리그램~1000 mg +십삼 톤~13 t +사백오십 그램~450 g +이 리터~2 L +오백 밀리리터~500 ml +일 점 오 리터~1.5 L +십 씨씨~10 cc +삼천 씨씨~3000 cc +칠십 밀리리터~70 ml +영하 십 도~-10 ° +이십오 도~25 ° +백 도~100 ° +삼십육점오 도~36.5 ° +삼십삼 평~33 py +백 제곱미터~100 m² +오십 평~50 py +십 제곱킬로미터~10 km² +오백 기가바이트~500 GB +십육 기가~16 GB +일 테라바이트~1 TB +이백오십육 메가바이트~256 MB +육십 헤르츠~60 Hz +백 메가헤르츠~100 MHz +백 킬로미터퍼시간~100 km/h +백퍼센트~100 % +오십 프로~50 % +삼십 점 오 퍼센트~30.5 % +십 미터퍼초~10 m/s +천알피엠~1000 rpm +이백이십 볼트~220 V +육십 와트~60 W +십 암페어~10 A +오천 밀리암페어~5000 mA +일점오 볼트~1.5 V \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..09c6b2841 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_money.txt @@ -0,0 +1,29 @@ +오천 원~₩5000 +만오천원~₩15000 +십이만삼천 원~₩123000 +백 원~₩100 +천백십일 원~₩1111 +육십만 원~₩600000 +백만 원~₩1000000 +삼백오십만 원~₩3500000 +천이백만 원~₩12000000 +일억 원~₩100000000 +십이억오천만 원~₩1250000000 +백억 원~₩10000000000 +오천억원~₩500000000000 +일조 원~₩1000000000000 +삼조오천억 원~₩3500000000000 +영원~₩0 +구십구 원~₩99 +만 원~₩10000 +일만 원~₩10000 +십오 달러~$15 +이십불~$20 +천오백 불~$1500 +백만 달러~$1000000 +오십 유로~€50 +천 엔~¥1000 +만 엔~¥10000 +백 파운드~£100 +이십 위안~¥20 +구천구백구십구원~₩9999 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..96fbb7005 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,17 @@ +영번째~0번째 +첫번째~1번째 +두번째~2번째 +세번째~3번째 +다섯번째~5번째 +아홉번째~9번째 +열번째~10번째 +열한번째~11번째 +열일곱번째~17번째 +스무번째~20번째 +스물두번째~22번째 +스물아홉번째~29번째 +서른번째~30번째 +사십번째~40번째 +오십번째~50번째 +오십삼번째~53번째 +백번째~100번째 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..605baf0c7 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_telephone.txt @@ -0,0 +1,7 @@ +공일공에일이삼사에오육칠팔~010-1234-5678 +영일영 구팔칠육 오사삼이~010-9876-5432 +공이에삼사오육에칠팔구공~02-3456-7890 +공삼일에구팔칠에육오사삼~031-987-6543 +공일공 일이삼사 오육칠팔~010-1234-5678 +공이 삼사오에육칠팔구~02-345-6789 +공일공일이삼사오육칠팔~01012345678 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..2700b6ccc --- /dev/null +++ b/tests/nemo_text_processing/ko/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,15 @@ +두시~2:00 +열두시~12:00 +두시 삼십분~2:30 +세시 삼분~3:03 +두시 반~2:30 +열두시 반~12:30 +두시 오초~2:00:05 +두시 삼십분 오초~2:30:05 +오전 두시~오전 2:00 +오후 네시 반~오후 4:30 +두시전~2:00 전 +두시십분후~2:10 후 +한시 십오분 삼십초~1:15:30 +네시 이분~4:02 +열한시 오십구분~11:59 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..dbde76d4e --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,68 @@ +1~일 +2~이 +3~삼 +123~백이십삼 +13000~만삼천 +9000~구천 +123000~십이만삼천 +123000012~일억이천삼백만십이 +1000000~백만 +100000000~일억 +1000000000000~일조 +100000000000000~백조 +20000000000001~이십조일 +800000000001001~팔백조천일 +82345670123135111~팔경이천삼백사십오조육천칠백일억이천삼백십삼만오천백십일 +9999999999999~구조구천구백구십구억구천구백구십구만구천구백구십구 +99999999999999~구십구조구천구백구십구억구천구백구십구만구천구백구십구 +999999999999999~구백구십구조구천구백구십구억구천구백구십구만구천구백구십구 +9999999999999999~구천구백구십구조구천구백구십구억구천구백구십구만구천구백구십구 +19~십구 +76~칠십육 +379~삼백칠십구 +850~팔백오십 +1004~천사 +8326~팔천삼백이십육 +10383~만삼백팔십삼 +34892~삼만사천팔백구십이 +573234~오십칠만삼천이백삼십사 +982010~구십팔만이천십 +2349023~이백삼십사만구천이십삼 +4303189~사백삼십만삼천백팔십구 +60321589~육천삼십이만천오백팔십구 +88234568~팔천팔백이십삼만사천오백육십팔 +792133923~칠억구천이백십삼만삼천구백이십삼 +187624689~일억팔천칠백육십이만사천육백팔십구 +2304050708~이십삼억사백오만칠백팔 +6436789729~육십사억삼천육백칠십팔만구천칠백이십구 +78234580257~칠백팔십이억삼천사백오십팔만이백오십칠 +987654321345~구천팔백칠십육억오천사백삼십이만천삼백사십오 +2345678901234~이조삼천사백오십육억칠천팔백구십만천이백삼십사 +35791357913579~삼십오조칠천구백십삼억오천칠백구십일만삼천오백칠십구 +470369258147036~사백칠십조삼천육백구십이억오천팔백십사만칠천삼십육 +5048258149517395~오천사십팔조이천오백팔십일억사천구백오십일만칠천삼백구십오 +67890123045607890~육경칠천팔백구십조천이백삼십억사천오백육십만칠천팔백구십 +-2~마이너스 이 +-93~마이너스 구십삼 +-90325~마이너스 구만삼백이십오 +-3234567~마이너스 삼백이십삼만사천오백육십칠 +휴대폰 번호는 0987654321입니다~휴대폰 번호는 영구팔칠육오사삼이일 입니다 +전화 번호는 01090817263입니다~전화 번호는 영일영구영팔일칠이육삼 입니다 +계좌 번호는 70501938462011입니다~계좌 번호는 칠영오영일구삼팔사육이영일일 입니다 +예약 번호는 907입니다~예약 번호는 구영칠 입니다 +예약 번호는 830-291입니다~예약 번호는 팔삼영이구일 입니다 +주문 번호는 4829.7301입니다~주문 번호는 사팔이구칠삼영일 입니다 +인증 번호는 000739입니다~인증 번호는 영영영칠삼구 입니다 +인증 번호는 204060입니다~인증 번호는 이영사영육영 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +연락처는 84502719입니다~연락처는 팔사오영이칠일구 입니다 +연락처가 77008899입니다~연락처가 칠칠영영팔팔구구 입니다 +연락처를 604 812 930으로 저장했습니다~연락처를 육영사팔일이구삼영 으로 저장했습니다 +번호는 12입니다~번호는 십이 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +접수 번호는 812709입니다~접수 번호는 팔일이칠영구 입니다 +신청 번호는 40682015입니다~신청 번호는 사영육팔이영일오 입니다 +확인 번호는 970045입니다~확인 번호는 구칠영영사오 입니다 +관리 번호는 25081973입니다~관리 번호는 이오영팔일구칠삼 입니다 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..c3e81a25c --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_date.txt @@ -0,0 +1,29 @@ +2024년~이천이십사년 +3월~삼월 +15일~십오일 +2024/01/30~이천이십사년 일월 삼십일 +10/30/2024~이천이십사년 시월 삼십일 +29/12/2000~이천년 십이월 이십구일 +2024/3/20~이천이십사년 삼월 이십일 +2024-07-23~이천이십사년 칠월 이십삼일 +1999.9.17~천구백구십구년 구월 십칠일 +기원전128년~기원전 백이십팔년 +기원후1390년~기원후 천삼백구십년 +28일수요일~이십팔일 수요일 +1900년대~천구백년대 +1월1일(월)〜3일(수)~일월 일일 월요일 부터 삼일 수요일 +5월10일(금)〜15일(수)~오월 십일 금요일 부터 십오일 수요일 +8월20일〜25일~팔월 이십일 부터 이십오일 +12월30일(토)〜1월2일(화)~십이월 삼십일 토요일 부터 일월 이일 화요일 +2월28일(목)〜3월3일(일)~이월 이십팔일 목요일 부터 삼월 삼일 일요일 +6월1일〜5일~유월 일일 부터 오일 +10월8일(화)〜10일(목)~시월 팔일 화요일 부터 십일 목요일 +1970〜1980년대~천구백칠십 부터 천구백팔십년대 +80〜90년대~팔십 부터 구십년대 +2010〜2020년대~이천십 부터 이천이십년대 +7월5〜9일(월〜금)~칠월 오 부터 구일 월요일부터금요일 +3월10〜15일(화〜일)~삼월 십 부터 십오일 화요일부터일요일 +11월1〜5일(수〜일)~십일월 일 부터 오일 수요일부터일요일 +2023년3월1일(수)〜6월12일(화)~이천이십삼년 삼월 일일 수요일 부터 유월 십이일 화요일 +2024년1월15일(월)〜2월10일(토)~이천이십사년 일월 십오일 월요일 부터 이월 십일 토요일 +2025년12월20일(토)〜2026년1월5일(월)~이천이십오년 십이월 이십일 토요일 부터 이천이십육년 일월 오일 월요일 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..d363c5bb2 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,28 @@ +-0.1~마이너스 영점일 +-0.5~마이너스 영점오 +-1.1~마이너스 일점일 +-2.5~마이너스 이점오 +-4.2~마이너스 사점이 +-11.99~마이너스 십일점구구 +-15.8~마이너스 십오점팔 +-25.3~마이너스 이십오점삼 +-30.8~마이너스 삼십점팔 +-72.4~마이너스 칠십이점사 +-100.5~마이너스 백점오 +0.1~영점일 +0.5~영점오 +1.1~일점일 +2.5~이점오 +4.2~사점이 +11.99~십일점구구 +15.8~십오점팔 +25.3~이십오점삼 +30.8~삼십점팔 +42.75~사십이점칠오 +72.4~칠십이점사 +100.5~백점오 +123.99~백이십삼점구구 +165.4~백육십오점사 +999.99~구백구십구점구구 +1000.01~천점영일 +123456.2234~십이만삼천사백오십육점이이삼사 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..4e09d0db2 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,20 @@ +a@hotmail.de~a 골뱅이 hotmail 닷 디이 +a@hotmail.fr~a 골뱅이 hotmail 닷 에프알 +a@hotmail.it~a 골뱅이 hotmail 닷 아이티 +a@aol.it~a 골뱅이 aol 닷 아이티 +a@msn.it~a 골뱅이 msn 닷 아이티 +abc@nvidia.app~abc 골뱅이 nvidia 닷 앱 +user01@gmail.co.kr~user영일 골뱅이 gmail 닷 씨오 닷 케이알 +nvidia.co.kr~nvidia 닷 씨오 닷 케이알 +1234-5678-9012-3456~신용카드 일이삼사 오육칠팔 구영일이 삼사오육 +2345-2222-3333-4444~신용카드 이삼사오 이이이이 삼삼삼삼 사사사사 +9090-1234-5555-9876~신용카드 구영구영 일이삼사 오오오오 구팔칠육 +카드 마지막 네자리 3456~카드 마지막 네자리 삼사오육 +카드 마지막 4자리 7890~카드 마지막 네자리 칠팔구영 +카드 끝자리 3456~카드 끝자리 삼사오육 +사진.jpg~사진 닷 제이피지 +사진.JPG~사진 닷 제이피지 +사진.png~사진 닷 피엔지 +사진.PNG~사진 닷 피엔지 +문서.pdf~문서 닷 피디에프 +문서.PDF~문서 닷 피디에프 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..65e5049b8 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,28 @@ +1/2~이분의 일 +-1/2~마이너스 이분의 일 +1과1/2~일과 이분의 일 +2와12/33~이와 삼십삼분의 십이 +-1과1/2~마이너스 일과 이분의 일 +마이너스1과1/2~마이너스 일과 이분의 일 +마이너스1과√1/2~마이너스 일과 이분의 루트 일 +-1과√1/2~마이너스 일과 이분의 루트 일 +1과√1/2~일과 이분의 루트 일 +1과1/√3~일과 루트 삼분의 일 +1과1/3~일과 삼분의 일 +1과√1/4~일과 사분의 루트 일 +3분의1~삼분의 일 +121분의3221~백이십일분의 삼천이백이십일 +이번 경기의 3/5이 중요하다~이번 경기의 오분의 삼이 중요하다 +전체 구역의 4/7이 통제되었다~전체 구역의 칠분의 사가 통제되었다 +설문 응답자의 9/10이 찬성했다~설문 응답자의 십분의 구가 찬성했다 +그 중 2/3은 성공했다~그 중 삼분의 이는 성공했다 +참가자의 5/8이 탈락했다~참가자의 팔분의 오가 탈락했다 +참가자의 6/7 이 통과했다~참가자의 칠분의 육 이 통과했다 +전체의 3/4 이 감소했다~전체의 사분의 삼 이 감소했다 +응답자의 2/5이 반대했다~응답자의 오분의 이가 반대했다 +학생의 7/9 이 합격했다~학생의 구분의 칠 이 합격했다 +전체의 1/2 이 남았다~전체의 이분의 일 이 남았다 +그 중 4/5이 성공했다~그 중 오분의 사가 성공했다 +전체의 5/6이 완료되었다~전체의 육분의 오가 완료되었다 +참가자의 3/8이 탈락했다~참가자의 팔분의 삼이 탈락했다 +응답자의 6/10 이 동의했다~응답자의 십분의 육 이 동의했다 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..3d24c4f0d --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,40 @@ +1kg~일 킬로그램 +12kg~십이 킬로그램 +0kg~영 킬로그램 +100g~백 그램 +500g~오백 그램 +1m~일 미터 +12km~십이 킬로미터 +5cm~오 센티미터 +15mm~십오 밀리미터 +1000km~천 킬로미터 +9999m~구천구백구십구 미터 +10L~십 리터 +1l~일 리터 +250ml~이백오십 밀리리터 +1mL~일 밀리리터 +123h~백이십삼 시간 +3s~삼 초 +60km/h~육십 킬로미터 퍼 시간 +1m/s~일 미터 퍼 초 +12kg/kg~십이 킬로그램 퍼 킬로그램 +1km/m~일 킬로미터 퍼 미터 +50W~오십 와트 +440Hz~사백사십 헤르츠 +300N~삼백 뉴턴 +120rpm~백이십 분당회전수 +100%~백 퍼센트 +30°~삼십 도 +0.5kg~영점오 킬로그램 +2.3km~이점삼 킬로미터 +12.5L~십이점오 리터 +3.14m~삼점일사 미터 +0.03m~영점영삼 미터 +1/2kg~이분의 일 킬로그램 +2/3km~삼분의 이 킬로미터 +5/8cm~팔분의 오 센티미터 +2과3/4L~이과 사분의 삼 리터 +10과1/2km/h~십과 이분의 일 킬로미터 퍼 시간 +-3/4km~마이너스 사분의 삼 킬로미터 +-3.1km~마이너스 삼점일 킬로미터 +-3km~마이너스 삼 킬로미터 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..7a22075ab --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_money.txt @@ -0,0 +1,64 @@ +₩2000~이천원 +₩2,000~이천원 +2000원~이천원 +KRW 1230000~백이십삼만원 +₩460000000000~사천육백억원 +₩30억~삼십억원 +₩30조~삼십조원 +₩45억~사십오억원 +₩15000~만오천원 +₩1~일원 +₩20~이십원 +₩18925000~천팔백구십이만오천원 +₩18,925,000~천팔백구십이만오천원 +₩18854~만팔천팔백오십사원 +₩18129~만팔천백이십구원 +₩0~영원 +₩7~칠원 +₩10~십원 +₩11~십일원 +₩21~이십일원 +₩99~구십구원 +200원~이백원 +999원~구백구십구원 +₩1,000~천원 +₩9,999~구천구백구십구원 +₩10,000~만원 +₩20,000~이만원 +₩100,000~십만원 +₩1,000,000~백만원 +₩2,500,000~이백오십만원 +₩12,345~만이천삼백사십오원 +₩1,234,567~백이십삼만사천오백육십칠원 +₩10,000,000~천만원 +₩23,456,000~이천삼백사십오만육천원 +₩100,000,000~일억원 +₩123,000,000~일억이천삼백만원 +₩2억~이억원 +₩12억~십이억원 +₩100억~백억원 +₩1,000억~천억원 +₩3,400억~삼천사백억원 +₩4조~사조원 +₩12조~십이조원 +KRW 1,000,000,000~십억원 +krw 2,345,600,000~이십삼억사천오백육십만원 +₩2,300,000,000~이십삼억원 +₩999,999,999~구억구천구백구십구만구천구백구십구원 +KRW 30,000,000~삼천만원 +krw 5000~오천원 +$ 0~영 달러 +$ 1~일 달러 +$ 200~이백 달러 +US$ 1,234,567~백이십삼만사천오백육십칠 달러 +$ 30억~삼십억 달러 +HK$ 300~삼백 홍콩 달러 +€500~오백 유로 +EUR 1,230,000~백이십삼만 유로 +¥2000~이천 엔 +JPY 1,230,000~백이십삼만 엔 +¥30조~삼십조 엔 +CAD 2,500~이천오백 캐나다 달러 +NZD 123,456~십이만삼천사백오십육 뉴질랜드 달러 +CHF 100~백 스위스 프랑 +AED 75~칠십오 아랍에미리트 디르함 diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..3544a2aeb --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,54 @@ +1번째~첫번째 +2번째~두번째 +3번째~세번째 +4번째~네번째 +5번째~다섯번째 +6번째~여섯번째 +7번째~일곱번째 +8번째~여덟번째 +9번째~아홉번째 +10번째~열번째 +11번째~열한번째 +12번째~열두번째 +13번째~열세번째 +14번째~열네번째 +15번째~열다섯번째 +16번째~열여섯번째 +17번째~열일곱번째 +18번째~열여덟번째 +19번째~열아홉번째 +20번째~스무번째 +21번째~스물한번째 +22번째~스물두번째 +23번째~스물세번째 +24번째~스물네번째 +25번째~스물다섯번째 +26번째~스물여섯번째 +27번째~스물일곱번째 +28번째~스물여덟번째 +29번째~스물아홉번째 +30번째~서른번째 +31번째~서른한번째 +32번째~서른두번째 +33번째~서른세번째 +34번째~서른네번째 +35번째~서른다섯번째 +36번째~서른여섯번째 +37번째~서른일곱번째 +38번째~서른여덟번째 +39번째~서른아홉번째 +100번째~백번째 +189번째~백팔십구번째 +1034번째~천삼십사번째 +7324번째~칠천삼백이십사번째 +23456번째~이만삼천사백오십육번째 +78903번째~칠만팔천구백삼번째 +345678번째~삼십사만오천육백칠십팔번째 +987654번째~구십팔만칠천육백오십사번째 +1000000번째~백만번째 +5678901번째~오백육십칠만팔천구백일번째 +89123456번째~팔천구백십이만삼천사백오십육번째 +62345098번째~육천이백삼십사만오천구십팔번째 +235067092번째~이억삼천오백육만칠천구십이번째 +876543210번째~팔억칠천육백오십사만삼천이백십번째 +1000000000번째~십억번째 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..a871e4f71 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt @@ -0,0 +1,39 @@ ++1 123-123-5678~국가번호 일 일이삼 일이삼 오육칠팔 +123-123-5678~일이삼 일이삼 오육칠팔 ++1-123-123-5678~국가번호 일 일이삼 일이삼 오육칠팔 ++1 (123)-123-5678~국가번호 일 일이삼 일이삼 오육칠팔 +(123)-123-5678~일이삼 일이삼 오육칠팔 +555.555.5555~오오오 오오오 오오오오 +(123) 123-5678~일이삼 일이삼 오육칠팔 +010-3713-7050~영일영 삼칠일삼 칠영오영 ++82 123-456-7890~국가번호 팔이 일이삼 사오육 칠팔구영 ++82-123-4567-8901~국가번호 팔이 일이삼 사오육칠 팔구영일 ++44-207-555-1234~국가번호 사사 이영칠 오오오 일이삼사 +123.456-7890~일이삼 사오육 칠팔구영 +123-456.7890~일이삼 사오육 칠팔구영 +(987)-654-3210~구팔칠 육오사 삼이일영 +(987) 654-3210~구팔칠 육오사 삼이일영 ++7 000-000-0000~국가번호 칠 영영영 영영영 영영영영 +000.000.0000~영영영 영영영 영영영영 +271-828-1828~이칠일 팔이팔 일팔이팔 +314-159-2653~삼일사 일오구 이육오삼 +(010) 123-4567~영일영 일이삼 사오육칠 ++358-123-456-7890~국가번호 삼오팔 일이삼 사오육 칠팔구영 ++1 800-555-0000~국가번호 일 팔영영 오오오 영영영영 +(800) 555-0000~팔영영 오오오 영영영영 ++12 345-678-9012~국가번호 일이 삼사오 육칠팔 구영일이 ++999 999-999-9999~국가번호 구구구 구구구 구구구 구구구구 +321.654.0987~삼이일 육오사 영구팔칠 ++82 010-1234-5678~국가번호 팔이 영일영 일이삼사 오육칠팔 +(999)-000-0000~구구구 영영영 영영영영 ++1-123.456.7890~국가번호 일 일이삼 사오육 칠팔구영 ++82-123.456-7890~국가번호 팔이 일이삼 사오육 칠팔구영 +111-222-3333~일일일 이이이 삼삼삼삼 +909-808-7070~구영구 팔영팔 칠영칠영 +(555)555-5555~오오오 오오오 오오오오 +02-123-4567~영이 일이삼 사오육칠 +02-1234-5678~영이 일이삼사 오육칠팔 +043-123-4567~영사삼 일이삼 사오육칠 +043-1234-5678~영사삼 일이삼사 오육칠팔 +(02) 1234-5678~영이 일이삼사 오육칠팔 +010 1234 5678~영일영 일이삼사 오육칠팔 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..2fb79402d --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_time.txt @@ -0,0 +1,25 @@ +오전10시30분~오전 열시 삼십분 +오전 10시30분~오전 열시 삼십분 +오전 10:30~오전 열시 삼십분 +오전 10:30:05~오전 열시 삼십분 오초 +오후 3시 반~오후 세시 반 +오후3시30분~오후 세시 삼십분 +오후 03:30~오후 세시 삼십분 +새벽 4시 5분~새벽 네시 오분 +새벽 04:05~새벽 네시 오분 +아침 7시~아침 일곱시 +낮 12시 15분~낮 열두시 십오분 +저녁 8시 45분~저녁 여덟시 사십오분 +밤 11시 55분 5초~밤 열한시 오십오분 오초 +밤 11:50:05~밤 열한시 오십분 오초 +정오~열두시 +자정~영시 +14:05~십사시 오분 +18:05~십팔시 오분 +23:00~이십삼시 +00:30:00~영시 삼십분 +24:03:38~이십사시 삼분 삼십팔초 +오전 0시 15분~오전 영시 십오분 +오후 12시 10분~오후 열두시 십분 +아침7시1분~아침 일곱시 일분 +저녁9시09분~저녁 아홉시 구분 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/test_cardinal.py b/tests/nemo_text_processing/ko/test_cardinal.py new file mode 100644 index 000000000..d7d1f74c7 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_cardinal.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer_ko = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_ko.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_date.py b/tests/nemo_text_processing/ko/test_date.py new file mode 100644 index 000000000..a0f8daa9f --- /dev/null +++ b/tests/nemo_text_processing/ko/test_date.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_date(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_decimal.py b/tests/nemo_text_processing/ko/test_decimal.py new file mode 100644 index 000000000..99cf64894 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_decimal.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_decimal(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_electronic.py b/tests/nemo_text_processing/ko/test_electronic.py new file mode 100644 index 000000000..d06099328 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_electronic.py @@ -0,0 +1,32 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer +from ..utils import parse_test_case_file + + +class TestElectronic: + normalizer_ko = Normalizer( + lang='ko', cache_dir='export/ko_tn_grammars_lower_cased', overwrite_cache=False, input_case='lower_cased' + ) + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_electronic(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ko/test_fraction.py b/tests/nemo_text_processing/ko/test_fraction.py new file mode 100644 index 000000000..7b8742da6 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_fraction.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFraction: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_fraction.txt')) + # @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_fraction(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_measure.py b/tests/nemo_text_processing/ko/test_measure.py new file mode 100644 index 000000000..ea053b281 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_measure.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_measure(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_money.py b/tests/nemo_text_processing/ko/test_money.py new file mode 100644 index 000000000..63ff5ca1b --- /dev/null +++ b/tests/nemo_text_processing/ko/test_money.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_money(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_ordinal.py b/tests/nemo_text_processing/ko/test_ordinal.py new file mode 100644 index 000000000..256a79249 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_ordinal.py @@ -0,0 +1,43 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestOrdinal: + normalizer_ko = Normalizer( + lang='ko', cache_dir='export/ko_tn_grammars_lower_cased', overwrite_cache=False, input_case='lower_cased' + ) + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_date(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input, verbose=True) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/ko/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..f11856232 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,74 @@ +#! /bin/sh + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests/ko"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNOrdinal() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testITNDecimal() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNFraction() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_fraction.txt + runtest $input +} + +testITNTime() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNDate() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNMoney() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_money.txt + runtest $input +} + +testITNTelephone() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_telephone.txt + runtest $input +} + +testITNMeasure() { + input=$TEST_DIR/data_inverse_text_normalization/test_cases_measure.txt + runtest $input +} + +# Remove all command-line arguments +shift $# + +# Load shUnit2 +. /workspace/shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..9adbc152b --- /dev/null +++ b/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh @@ -0,0 +1,83 @@ +#! /bin/sh +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read written spoken <<< $testcase + # replace non breaking space with breaking space + # Use below if postprocessor is not used. Comment if it is used + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + # Use below if postprocessor is used. Comment if it is not used + # denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration_pp.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + + +testTNCardinal() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNOrdinalText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testTNDecimalalText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +testTNFractionText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_fraction.txt + runtest $input +} + +testTNDateText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_date.txt + runtest $input +} + +testTNTimeText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_time.txt + runtest $input +} + +testTNMoneyText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_money.txt + runtest $input +} + +testTNTelephoneText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_telephone.txt + runtest $input +} + +testTNMeasureText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_measure.txt + runtest $input +} + +testTNElectronicText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_electronic.txt + runtest $input +} + +# Remove all command-line arguments +shift $# + +# Load shUnit2 +. /workspace/shunit2/shunit2 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/test_telephone.py b/tests/nemo_text_processing/ko/test_telephone.py new file mode 100644 index 000000000..949fdc068 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_telephone.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_telephone(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ko/test_time.py b/tests/nemo_text_processing/ko/test_time.py new file mode 100644 index 000000000..9ba942772 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_time.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + normalizer_ko = Normalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='lower_cased') + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_time(self, test_input, expected): + preds = self.normalizer_ko.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='ko', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('ko/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/mr/__init__.py b/tests/nemo_text_processing/mr/__init__.py new file mode 100644 index 000000000..d9155f923 --- /dev/null +++ b/tests/nemo_text_processing/mr/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..e5ccff889 --- /dev/null +++ b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,33 @@ +शून्य~० +उणे शंभर~-१०० +उणे सत्तावीस~-२७ +तीनशे एक~३०१ +सातशे~७०० +सातशे बावीस~७२२ +तीन हजार~३००० +आठ हजार तीन~८००३ +पाच हजार नव्वद~५०९० +उणे दोन हजार चाळीस~-२०४० +एक हजार सातशे अडुसष्ठ~१७६८ +नऊ हजार दोनशे~९२०० +तीन हजार एकशे~३१०० +एकतीसशे~३१०० +अठ्ठावन्नशे एक~५८०१ +उणे बावन्नशे~-५२०० +पंचाहत्तरशे सोळा~७५१६ +तेहतीस हजार~३३००० +सत्त्याण्णव हजार तीन~९७००३ +एक्याऐंशी हजार एक्याऐंशी~८१०८१ +सतरा हजार सातशे~१७७०० +उणे छप्पन हजार~-५६००० +उणे त्र्याऐंशी हजार सातशे पाच~-८३७०५ +चार लाख अठरा हजार~४१८००० +सात लाख आठ~७००००८ +सत्तावन्न लाख अठ्ठ्यात्तर हजार तीनशे नव्व्याण्णव~५७७८३९९ +उणे नऊ लाख दहा~-९०००१० +आठ कोटी एकतीस लाख बारा हजार नऊशे त्रेचाळीस~८३११२९४३ +एक्याण्णव कोटी सत्तर लाख~९१७०००००० +तेहतीस कोटी तेहतीस लाख बासष्ट हजार~३३३३६२००० +तेहतीस कोटी तेहतीस लाख पंचवीस~३३३३०००२५ +उणे तीन कोटी तेहतीस लाख~-३३३००००० +उणे तेहतीस कोटी तेहतीस लाख~-३३३३००००० \ No newline at end of file diff --git a/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..c4e0455d2 --- /dev/null +++ b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_date.txt @@ -0,0 +1,29 @@ +दहा जून दोन हजार~१० जून २००० +जानेवारी तेरा दोन हजार तेवीस~जानेवारी १३ २०२३ +जानेवारी दहा~जानेवारी १० +जानेवारी दोन हजार~जानेवारी २००० +पंधरा फेब्रुवारी अठराशे सतरा~१५ फेब्रुवारी १८१७ +इसवी सन बाराशे सत्याहत्तर~इ.स. १२७७ +इसवी सन पूर्व एकोणीसशे नव्व्याण्णव~इ.स.पू. १९९९ +ऑक्टोबर तेवीस सातशे आठ~ऑक्टोबर २३ ७०८ +फेब्रुवारी एकोणतीस~फेब्रुवारी २९ +इसवी सन सतराशे~इ.स. १७०० +मार्च बावीस एकोणीसशे अठ्ठ्याण्णव~मार्च २२ १९९८ +एप्रिल एक अकराशे अकरा~एप्रिल १ ११११ +बावीस मे एकोणीसशे एकसष्ट~२२ मे १९६१ +एकतीस जुलै एक हजार~३१ जुलै १००० +ऑगस्ट सात~ऑगस्ट ७ +सप्टेंबर सत्तावीस दोन हजार सहा~सप्टेंबर २७ २००६ +नऊ ऑक्टोबर दोन हजार दोन~९ ऑक्टोबर २००२ +नोव्हेंबर सोळाशे सत्तेचाळीस~नोव्हेंबर १६४७ +एकतीस डिसेंबर एकोणीसशे नव्व्याण्णव~३१ डिसेंबर १९९९ +मार्च~मार्च +एप्रिल अठराशे एक~एप्रिल १८०१ +इसवी सन पूर्व चारशे~इ.स.पू. ४०० +इसवी सन दोन हजार दहा~इ.स. २०१० +जुलै पंधरा~जुलै १५ +वीस ऑगस्ट एकोणीसशे~२० ऑगस्ट १९०० +चौदा सप्टेंबर~१४ सप्टेंबर +ऑक्टोबर चार~ऑक्टोबर ४ +तेरा नोव्हेंबर दोन हजार तेवीस~१३ नोव्हेंबर २०२३ +डिसेंबर पंचवीस~डिसेंबर २५ \ No newline at end of file diff --git a/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..2ec11fefc --- /dev/null +++ b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_decimal.txt @@ -0,0 +1,29 @@ +तेहतीस पूर्णांक तीन~३३.३ +चारशे पूर्णांक शून्य पाच~४००.०५ +आठशे पूर्णांक शून्य तीन एक~८००.०३१ +दोन हजार पूर्णांक आठ एक दोन पाच~२०००.८१२५ +चार लाख~४ लाख +उणे बाहत्तर लाख~-७२ लाख +तेहतीस लाख सत्तावीस हजार आठशे वीस पूर्णांक नऊ~३३२७८२०.९ +एकेचाळीस पूर्णांक तीन लाख~४१.३ लाख +उणे एक्याण्णव पूर्णांक चार कोटी~-९१.४ कोटी +दहा पूर्णांक पाच सहा~१०.५६ +सात पूर्णांक सात~७.७ +पंचाहत्तर कोटी~७५ कोटी +उणे बारा पूर्णांक एक पाच कोटी~-१२.१५ कोटी +बावन्न पूर्णांक एक शून्य चार आठ पाच~५२.१०४८५ +उणे साठ हजार पूर्णांक नऊ नऊ नऊ शून्य~-६००००.९९९० +शून्य पूर्णांक आठ सात सहा~०.८७६ +उणे शून्य पूर्णांक एक दोन तीन~-०.१२३ +शंभर पूर्णांक नऊ~१००.९ +नव्व्याण्णव पूर्णांक नऊ नऊ नऊ~९९.९९९ +सतरा लाख~१७ लाख +उणे दोन पूर्णांक आठ एक दोन~-२.८१२ +सदुसष्ट पूर्णांक चार कोटी~६७.४ कोटी +उणे ऐंशी पूर्णांक एक सहा नऊ आठ लाख~-८०.१६९८ लाख +शंभर पूर्णांक एक दोन तीन चार पाच~१००.१२३४५ +शंभर कोटी~१०० कोटी +उणे सत्तर पूर्णांक पाच~-७०.५ +आठ पूर्णांक सहा लाख~८.६ लाख +पाच हजार दोनशे अडतीस पूर्णांक चार सहा~५२३८.४६ +चाळीस कोटी~४० कोटी \ No newline at end of file diff --git a/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..ea31ca0a5 --- /dev/null +++ b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_time.txt @@ -0,0 +1,29 @@ +साडे चार~०४:३० +साडे अकरा~११:३० +सव्वा आठ~०८:१५ +सव्वा बारा~१२:१५ +पावणे सहा~०५:४५ +पावणे दहा~०९:४५ +अकराला पाच मिनिटे~१०:५५ +आठला सात मिनिटे~०७:५३ +अकरा वाजून दोन मिनिटे~११:०२ +अकराला दोन मिनिट~१०:५८ +बारा वाजून पाच मिनिटे~१२:०५ +एक वाजून सात मिनिटे~०१:०७ +दीड~०१:३० +अडीच~०२:३० +नऊ वाजून वीस मिनिटे~०९:२० +सातला बारा मिनिटे~०६:४८ +पावणे अकरा~१०:४५ +दहा वाजून दहा मिनिटे~१०:१० +एकला एक मिनिट~१२:५९ +बारा वाजून पन्नास मिनिटे~१२:५० +चार वाजून पंचेचाळीस मिनिटे~०४:४५ +पावणे पाच~०४:४५ +पावणे एक~१२:४५ +सव्वा एक~०१:१५ +दहाला एक मिनिट~०९:५९ +सहा वाजून एक मिनिट~०६:०१ +नऊ वाजून तीस मिनिटे~०९:३० +दहाला पंधरा मिनिटे~०९:४५ +दोन वाजून अठ्ठेचाळीस मिनिटे~०२:४८ \ No newline at end of file diff --git a/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..3b7ef04e3 --- /dev/null +++ b/tests/nemo_text_processing/mr/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,11 @@ +~ +, मी~, मी +म ~म +मी~मी +आम्ही~आम्ही +तिथे~तिथे +थोडे~थोडे +आई~आई +वडील~वडील +झाड~झाड +पिणे~पिणे \ No newline at end of file diff --git a/tests/nemo_text_processing/mr/test_cardinal.py b/tests/nemo_text_processing/mr/test_cardinal.py new file mode 100644 index 000000000..588b7ebba --- /dev/null +++ b/tests/nemo_text_processing/mr/test_cardinal.py @@ -0,0 +1,33 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestPreprocess: + + inverse_normalizer_mr = InverseNormalizer(lang='mr', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('mr/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_mr.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/mr/test_date.py b/tests/nemo_text_processing/mr/test_date.py new file mode 100644 index 000000000..b58c1e532 --- /dev/null +++ b/tests/nemo_text_processing/mr/test_date.py @@ -0,0 +1,32 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + inverse_normalizer_mr = InverseNormalizer(lang='mr', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('mr/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_mr.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/mr/test_decimal.py b/tests/nemo_text_processing/mr/test_decimal.py new file mode 100644 index 000000000..e6f7d2d41 --- /dev/null +++ b/tests/nemo_text_processing/mr/test_decimal.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDecimal: + inverse_normalizer_mr = InverseNormalizer(lang='mr', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('mr/data_inverse_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_mr.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/mr/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/mr/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..9166db713 --- /dev/null +++ b/tests/nemo_text_processing/mr/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,49 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/mr/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testITNDecimal() { + input=$PROJECT_DIR/mr/data_inverse_text_normalization/test_cases_decimal.txt + runtest $input +} + +testITNTime() { + input=$PROJECT_DIR/mr/data_inverse_text_normalization/test_cases_time.txt + runtest $input +} + +testITNDate() { + input=$PROJECT_DIR/mr/data_inverse_text_normalization/test_cases_date.txt + runtest $input +} + +testITNWord() { + input=$PROJECT_DIR/mr/data_inverse_text_normalization/test_cases_word.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/mr/test_time.py b/tests/nemo_text_processing/mr/test_time.py new file mode 100644 index 000000000..5571a5349 --- /dev/null +++ b/tests/nemo_text_processing/mr/test_time.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + inverse_normalizer_mr = InverseNormalizer(lang='mr', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('mr/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_mr.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/mr/test_word.py b/tests/nemo_text_processing/mr/test_word.py new file mode 100644 index 000000000..263fe01a3 --- /dev/null +++ b/tests/nemo_text_processing/mr/test_word.py @@ -0,0 +1,32 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + inverse_normalizer_mr = InverseNormalizer(lang='mr', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('mr/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer_mr.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..be8057d71 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,121 @@ +0~zero +1~um +2~dois +3~três +4~quatro +5~cinco +6~seis +7~sete +8~oito +9~nove +10~dez +11~onze +12~doze +13~treze +14~catorze +15~quinze +16~dezesseis +17~dezessete +18~dezoito +19~dezenove +20~vinte +21~vinte e um +22~vinte e dois +23~vinte e três +24~vinte e quatro +25~vinte e cinco +26~vinte e seis +27~vinte e sete +28~vinte e oito +29~vinte e nove +30~trinta +40~quarenta +50~cinquenta +60~sessenta +70~setenta +80~oitenta +90~noventa +100~cem +101~cento e um +102~cento e dois +110~cento e dez +120~cento e vinte +130~cento e trinta +200~duzentos +300~trezentos +400~quatrocentos +500~quinhentos +600~seiscentos +700~setecentos +800~oitocentos +900~novecentos +1000~mil +1 000~mil +1.000~mil +1010~mil e dez +1020~mil e vinte +1100~mil e cem +1110~mil cento e dez +1111~mil cento e onze +2000~dois mil +2002~dois mil e dois +2010~dois mil e dez +2020~dois mil e vinte +2100~dois mil e cem +2110~dois mil cento e dez +2111~dois mil cento e onze +10000~dez mil +10 000~dez mil +10.000~dez mil +100000~cem mil +100 000~cem mil +100.000~cem mil +1 000 000~um milhão +1.000.000~um milhão +2.000.000~dois milhões +1.000.000.000~um bilhão +1000000000~um bilhão +2.000.000.000~dois bilhões +2000000000~dois bilhões +3 000 000 000 000~três trilhões +3.000.000.000.000~três trilhões +1001~mil e um +1010~mil e dez +1100~mil e cem +1101~mil cento e um +1111~mil cento e onze +1999~mil novecentos e noventa e nove +100000~cem mil +100001~cem mil e um +101000~cento e um mil +101001~cento e um mil e um +110000~cento e dez mil +111000~cento e onze mil +111111~cento e onze mil cento e onze +1001000~um milhão e mil +1001001~um milhão mil e um +1010000~um milhão e dez mil +1010101~um milhão dez mil cento e um +1100000~um milhão e cem mil +1110000~um milhão cento e dez mil +1001010101~um bilhão um milhão dez mil cento e um +1010101010~um bilhão dez milhões cento e um mil e dez +1234567890~um bilhão duzentos e trinta e quatro milhões quinhentos e sessenta e sete mil oitocentos e noventa +987654321~novecentos e oitenta e sete milhões seiscentos e cinquenta e quatro mil trezentos e vinte e um +999999999~novecentos e noventa e nove milhões novecentos e noventa e nove mil novecentos e noventa e nove +2000000001~dois bilhões e um +3000001000~três bilhões e mil +4000100000~quatro bilhões e cem mil +5000000100~cinco bilhões e cem +6001000000~seis bilhões e um milhão +1000000000000~um trilhão +1000000000001~um trilhão e um +1230000000000~um trilhão duzentos e trinta bilhões +3004005006007~três trilhões quatro bilhões cinco milhões seis mil e sete +1000001~um milhão e um +1001100~um milhão mil e cem +1001110~um milhão mil cento e dez +47701~quarenta e sete mil setecentos e um +394506~trezentos e noventa e quatro mil quinhentos e seis +3022110709~três bilhões vinte e dois milhões cento e dez mil setecentos e nove +302210709~trezentos e dois milhões duzentos e dez mil setecentos e nove \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..0338d420c --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_date.txt @@ -0,0 +1,26 @@ +15/03/2024~quinze de março de dois mil e vinte e quatro +01/01/2000~primeiro de janeiro de dois mil +31/12/1999~trinta e um de dezembro de mil novecentos e noventa e nove +15.03.2024~quinze de março de dois mil e vinte e quatro +15-03-2024~quinze de março de dois mil e vinte e quatro +1/5/2025~primeiro de maio de dois mil e vinte e cinco +07/08/2010~sete de agosto de dois mil e dez +9/6/2024~nove de junho de dois mil e vinte e quatro +2/3/2000~dois de março de dois mil +29/02/2024~vinte e nove de fevereiro de dois mil e vinte e quatro +25/12/2023~vinte e cinco de dezembro de dois mil e vinte e três +31/01/2024~trinta e um de janeiro de dois mil e vinte e quatro +06/09/2024~seis de setembro de dois mil e vinte e quatro +10/10/2010~dez de outubro de dois mil e dez +28.02.2023~vinte e oito de fevereiro de dois mil e vinte e três +01-06-1995~primeiro de junho de mil novecentos e noventa e cinco +15 de março de 2024~quinze de março de dois mil e vinte e quatro +1 de janeiro de 2000~primeiro de janeiro de dois mil +10 de Dezembro de 1999~dez de dezembro de mil novecentos e noventa e nove +2024-03-15~quinze de março de dois mil e vinte e quatro +2024-3-5~cinco de março de dois mil e vinte e quatro +2024-12-25~vinte e cinco de dezembro de dois mil e vinte e quatro +03/15/2024~quinze de março de dois mil e vinte e quatro +2024/03/15~quinze de março de dois mil e vinte e quatro +2024.03.15~quinze de março de dois mil e vinte e quatro +03/04/2024~três de abril de dois mil e vinte e quatro diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..0bea9be23 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,58 @@ +0,1~zero vírgula um +0,2~zero vírgula dois +0,5~zero vírgula cinco +0,9~zero vírgula nove +0,01~zero vírgula zero um +0,02~zero vírgula zero dois +0,05~zero vírgula zero cinco +0,10~zero vírgula um zero +0,11~zero vírgula um um +0,15~zero vírgula um cinco +0,20~zero vírgula dois zero +0,25~zero vírgula dois cinco +0,50~zero vírgula cinco zero +0,99~zero vírgula nove nove +1,1~um vírgula um +1,2~um vírgula dois +1,5~um vírgula cinco +1,10~um vírgula um zero +1,15~um vírgula um cinco +1,20~um vírgula dois zero +1,26~um vírgula dois seis +1,33~um vírgula três três +1,50~um vírgula cinco zero +3,141~três vírgula um quatro um +3,256~três vírgula dois cinco seis +3,999~três vírgula nove nove nove +3,1415~três vírgula um quatro um cinco +3,1001~três vírgula um zero zero um +3,014~três vírgula zero um quatro +3,0141~três vírgula zero um quatro um +3,1005~três vírgula um zero zero cinco +3,1050~três vírgula um zero cinco zero +-1,2~menos um vírgula dois +-1,26~menos um vírgula dois seis +-3,5~menos três vírgula cinco +-0,5~menos zero vírgula cinco +1,2 milhões~um vírgula dois milhões +1,5 milhões~um vírgula cinco milhões +1,25 milhões~um vírgula dois cinco milhões +2,5 bilhões~dois vírgula cinco bilhões +3,75 bilhões~três vírgula sete cinco bilhões +0,001~zero vírgula zero zero um +0,0001~zero vírgula zero zero zero um +1,001~um vírgula zero zero um +1,010~um vírgula zero um zero +1,100~um vírgula um zero zero +10,01~dez vírgula zero um +10,001~dez vírgula zero zero um +100,5~cem vírgula cinco +100,05~cem vírgula zero cinco +3,14~três vírgula um quatro +3,141~três vírgula um quatro um +3,1415~três vírgula um quatro um cinco +3,14159~três vírgula um quatro um cinco nove +1,1234567~um vírgula um dois três quatro cinco seis sete +3,1415926535~três vírgula um quatro um cinco nove dois seis cinco três cinco +12,27~doze vírgula dois sete +87,69~oitenta e sete vírgula seis nove diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..3aee0be70 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,13 @@ +test@gmail.com~t e s t arroba gmail ponto com +a.bc@gmail.com~a ponto b c arroba gmail ponto com +admin@company.com.br~a d m i n arroba c o m p a n y ponto com ponto br +www.google.com~w w w ponto google ponto com +https://www.nvidia.com~h t t p s dois pontos barra barra w w w ponto nvidia ponto com +http://site.com.br~h t t p dois pontos barra barra s i t e ponto com ponto br +nvidia.com~nvidia ponto com +@usuario~arroba usuario +mail@google.com~m a i l arroba google ponto com +support@microsoft.com~s u p p o r t arroba microsoft ponto com +https://www.amazon.com.br~h t t p s dois pontos barra barra w w w ponto amazon ponto com ponto br +u.s.e.r@facebook.com~u ponto s ponto e ponto r arroba facebook ponto com +api.netflix.com~a p i ponto netflix ponto com diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..e59642bac --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,22 @@ +1/2~um meio +1/3~um terço +1/4~um quarto +2/3~dois terços +3/4~três quartos +1/5~um quinto +2/5~dois quintos +1/6~um sexto +5/6~cinco sextos +1/8~um oitavo +3/8~três oitavos +7/8~sete oitavos +1/10~um décimo +3/10~três décimos +3/11~três onze avos +5/13~cinco treze avos +1/100~um centésimo +1/1000~um milésimo +1 1/2~um e um meio +2 1/4~dois e um quarto +3 2/3~três e dois terços +47701/913~quarenta e sete mil setecentos e um novecentos e treze avos \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..34a082108 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,18 @@ +200 g~duzentos gramas +1 kg~um quilo~um quilograma +5 kg~cinco quilos~cinco quilogramas +200 m~duzentos metros +1 km~um quilômetro +5 km~cinco quilômetros +100 m~cem metros +1 l~um litro +2 l~dois litros +500 ml~quinhentos mililitros +1 m²~um metro quadrado +10 m²~dez metros quadrados +25°C~vinte e cinco graus celsius +-5°C~menos cinco graus celsius +1 h~uma hora +2 h~duas horas +30 min~trinta minutos +45 s~quarenta e cinco segundos diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..ea0680741 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_money.txt @@ -0,0 +1,25 @@ +R$ 1~um real +R$ 12~doze reais +R$ 100~cem reais +R$ 200~duzentos reais +R$ 12,05~doze reais e cinco centavos +R$ 1,01~um real e um centavo +R$ 199,99~cento e noventa e nove reais e noventa e nove centavos +R$ 0,20~vinte centavos +R$ 0,25~vinte e cinco centavos +R$ 0,50~cinquenta centavos +$ 1~um dólar +$ 12~doze dólares +$ 12,05~doze dólares e cinco centavos +$ 29,50~vinte e nove dólares e cinquenta centavos +$ 75,63~setenta e cinco dólares e sessenta e três centavos~setenta e cinco dólares com sessenta e três centavos +€ 1~um euro +€ 12~doze euros +€ 12,05~doze euros e cinco centavos +R$ 1000~mil reais +R$ 1000000~um milhão de reais +$ 1000~mil dólares +$ 1000000~um milhão de dólares +R$ 181809~cento e oitenta e um mil oitocentos e nove reais +R$ 181819~cento e oitenta e um mil oitocentos e dezenove reais +R$ 1811605~um milhão oitocentos e onze mil seiscentos e cinco reais diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..f9a58a9ce --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,39 @@ +1º~primeiro +2º~segundo +3º~terceiro +4º~quarto +5º~quinto +6º~sexto +7º~sétimo +8º~oitavo +9º~nono +10º~décimo +11º~décimo primeiro +12º~décimo segundo +13º~décimo terceiro +20º~vigésimo +21º~vigésimo primeiro +22º~vigésimo segundo +23º~vigésimo terceiro +100º~centésimo +111º~centésimo décimo primeiro +134º~centésimo trigésimo quarto +1ª~primeira +2ª~segunda +3ª~terceira +4ª~quarta +5ª~quinta +6ª~sexta +7ª~sétima +8ª~oitava +9ª~nona +10ª~décima +11ª~décima primeira +12ª~décima segunda +13ª~décima terceira +20ª~vigésima +21ª~vigésima primeira +22ª~vigésima segunda +23ª~vigésima terceira +100ª~centésima +11ª casa~décima primeira casa \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..a5b085469 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_telephone.txt @@ -0,0 +1,7 @@ +(11) 99999-8888~um um nove nove nove nove nove oito oito oito oito +11 3333-4444~um um três três três três quatro quatro quatro quatro +555-1234~cinco cinco cinco um dois três quatro +99999-1234~nove nove nove nove nove um dois três quatro ++55 (11) 3333-4444~mais cinco cinco um um três três três três quatro quatro quatro quatro +192.168.1.1~um nove dois ponto um seis oito ponto um ponto um +(11) 3333-4444 ext. 12~um um três três três três quatro quatro quatro quatro extensão um dois \ No newline at end of file diff --git a/tests/nemo_text_processing/pt/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..b9ae642e7 --- /dev/null +++ b/tests/nemo_text_processing/pt/data_text_normalization/test_cases_time.txt @@ -0,0 +1,30 @@ +14:30~catorze horas e trinta +14h30~catorze horas e trinta +14h~catorze horas +09:05~nove horas e cinco +00:15~zero horas e quinze +23:59~vinte e três horas e cinquenta e nove +12:00~doze horas +14.30~catorze horas e trinta +08:00~oito horas +00:00~zero horas +9:30~nove horas e trinta +14H30~catorze horas e trinta +6h05~seis horas e cinco +23:00~vinte e três horas +10:10~dez horas e dez +7h~sete horas +00h~zero horas +12h30~doze horas e trinta +14:30:05~catorze horas e trinta minutos e cinco segundos +09:00:31~nove horas e zero minutos e trinta e um segundos +14:30:00~catorze horas e trinta minutos e zero segundos +09:00:00~nove horas e zero minutos e zero segundos +11:00 da manhã~onze horas da manhã +3:30 da tarde~três horas e trinta da tarde +15h da tarde~três horas da tarde +16:00 da tarde~quatro horas da tarde +14:30:05 da tarde~duas horas e trinta minutos e cinco segundos da tarde +21:18:14 da manhã~vinte e uma horas e dezoito minutos e catorze segundos da manhã +23:18 da tarde~vinte e três horas e dezoito da tarde +22h09 da tarde~vinte e duas horas e nove da tarde diff --git a/tests/nemo_text_processing/pt/test_cardinal.py b/tests/nemo_text_processing/pt/test_cardinal.py index bfe7d82d0..901bd7008 100644 --- a/tests/nemo_text_processing/pt/test_cardinal.py +++ b/tests/nemo_text_processing/pt/test_cardinal.py @@ -13,9 +13,11 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -29,3 +31,12 @@ class TestCardinal: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_date.py b/tests/nemo_text_processing/pt/test_date.py index 88b5a50eb..56a987e8f 100644 --- a/tests/nemo_text_processing/pt/test_date.py +++ b/tests/nemo_text_processing/pt/test_date.py @@ -13,9 +13,11 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -28,3 +30,12 @@ class TestDate: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_decimal.py b/tests/nemo_text_processing/pt/test_decimal.py index 4fd77295e..b66485a9b 100644 --- a/tests/nemo_text_processing/pt/test_decimal.py +++ b/tests/nemo_text_processing/pt/test_decimal.py @@ -13,9 +13,11 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -28,3 +30,12 @@ class TestDecimal: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_electronic.py b/tests/nemo_text_processing/pt/test_electronic.py index 9e340471f..9f462c5db 100644 --- a/tests/nemo_text_processing/pt/test_electronic.py +++ b/tests/nemo_text_processing/pt/test_electronic.py @@ -13,9 +13,11 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -28,3 +30,12 @@ class TestElectronic: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_fraction.py b/tests/nemo_text_processing/pt/test_fraction.py new file mode 100644 index 000000000..16e6c5f30 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_fraction.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestFraction: + normalizer = Normalizer( + lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased', post_process=True + ) + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" diff --git a/tests/nemo_text_processing/pt/test_measure.py b/tests/nemo_text_processing/pt/test_measure.py index 892b45962..603c967f4 100644 --- a/tests/nemo_text_processing/pt/test_measure.py +++ b/tests/nemo_text_processing/pt/test_measure.py @@ -14,9 +14,11 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -29,3 +31,15 @@ class TestMeasure: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + if isinstance(expected, list): + assert pred in expected + else: + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_money.py b/tests/nemo_text_processing/pt/test_money.py index 40c682fe9..d440a93e8 100644 --- a/tests/nemo_text_processing/pt/test_money.py +++ b/tests/nemo_text_processing/pt/test_money.py @@ -14,9 +14,11 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -29,3 +31,15 @@ class TestMoney: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + if isinstance(expected, list): + assert pred in expected + else: + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_ordinal.py b/tests/nemo_text_processing/pt/test_ordinal.py index 19acfbaee..c2e7dfb71 100644 --- a/tests/nemo_text_processing/pt/test_ordinal.py +++ b/tests/nemo_text_processing/pt/test_ordinal.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer from ..utils import CACHE_DIR, parse_test_case_file @@ -29,3 +30,12 @@ class TestOrdinal: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh index 74d8ddafd..511411e90 100755 --- a/tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/pt/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/pt/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/pt/test_sparrowhawk_normalization.sh new file mode 100755 index 000000000..c9f39aaa7 --- /dev/null +++ b/tests/nemo_text_processing/pt/test_sparrowhawk_normalization.sh @@ -0,0 +1,91 @@ +#! /bin/sh + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read -a testcase_tokenized <<< $testcase + written=${testcase_tokenized[0]} + # only tests against first possible option when there are multiple shortest paths + spoken=${testcase_tokenized[1]} + + # replace non breaking space with breaking space + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +testTNCardinal() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNDecimal() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +testTNOrdinal() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testTNFraction() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_fraction.txt + runtest $input +} + +testTNDate() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_date.txt + runtest $input +} + +testTNTime() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_time.txt + runtest $input +} + +testTNMeasure() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_measure.txt + runtest $input +} + +testTNMoney() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_money.txt + runtest $input +} + +# testTNWhitelist() { +# input=$PROJECT_DIR/pt/data_text_normalization/test_cases_whitelist.txt +# runtest $input +# } + +testTNTelephone() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_telephone.txt + runtest $input +} + +testTNElectronic() { + input=$PROJECT_DIR/pt/data_text_normalization/test_cases_electronic.txt + runtest $input +} + +# testTNWord() { +# input=$PROJECT_DIR/pt/data_text_normalization/test_cases_word.txt +# runtest $input +# } + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tests/nemo_text_processing/pt/test_telephone.py b/tests/nemo_text_processing/pt/test_telephone.py index 6d36e9db2..f69f0b5a3 100644 --- a/tests/nemo_text_processing/pt/test_telephone.py +++ b/tests/nemo_text_processing/pt/test_telephone.py @@ -14,9 +14,11 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file @@ -29,3 +31,12 @@ class TestTelephone: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_time.py b/tests/nemo_text_processing/pt/test_time.py index 7a556b36b..2a1a8f454 100644 --- a/tests/nemo_text_processing/pt/test_time.py +++ b/tests/nemo_text_processing/pt/test_time.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer from ..utils import CACHE_DIR, parse_test_case_file @@ -28,3 +29,12 @@ class TestTime: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer(lang='pt', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('pt/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/pt/test_whitelist.py b/tests/nemo_text_processing/pt/test_whitelist.py index 0f8884b53..399f191da 100644 --- a/tests/nemo_text_processing/pt/test_whitelist.py +++ b/tests/nemo_text_processing/pt/test_whitelist.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/pt/test_word.py b/tests/nemo_text_processing/pt/test_word.py index 2ad54b15e..cd3cc5d88 100644 --- a/tests/nemo_text_processing/pt/test_word.py +++ b/tests/nemo_text_processing/pt/test_word.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/ru/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/ru/test_sparrowhawk_inverse_text_normalization.sh index 1c975e070..6df536467 100644 --- a/tests/nemo_text_processing/ru/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/ru/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/rw/__init__.py b/tests/nemo_text_processing/rw/__init__.py new file mode 100644 index 000000000..4f53d71f2 --- /dev/null +++ b/tests/nemo_text_processing/rw/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/rw/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..baca7cbe4 --- /dev/null +++ b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,57 @@ +1~rimwe +2~kabiri +3~gatatu +4~kane +5~gatanu +6~gatandatu +7~karindwi +8~umunani +9~icyenda +10~icumi +002~zeru zeru kabiri +11~cumi na rimwe +12~cumi na kabiri +13~cumi na gatatu +2,3,4,5~kabiri gatatu kane gatanu +14~cumi na kane +15~cumi na gatanu +16~cumi na gatandatu +17~cumi na karindwi +18~cumi n'umunani +19~cumi n'icyenda +20~makumyabiri +89~mirongo inani n'icyenda +123~ijana na makumyabiri na gatatu +730~magana arindwi na mirongo itatu +100~ijana +2004~ibihumbi bibiri na kane +9041~ibihumbi icyenda na mirongo ine na rimwe +5324~ibihumbi bitanu na magana atatu na makumyabiri na kane +8567~ibihumbi umunani na magana atanu na mirongo itandatu na karindwi +10000~ibihumbi icumi +14000~ibihumbi cumi na bine +24404~ibihumbi makumyabiri na bine na magana ane na kane +9000~ibihumbi icyenda +9700~ibihumbi icyenda na magana arindwi +250~magana abiri na mirongo itanu +367~magana atatu na mirongo itandatu na karindwi +90104~ibihumbi mirongo icyenda n'ijana na kane +111001~ibihumbi ijana na cumi na kimwe na rimwe +10999~ibihumbi icumi na magana cyenda na mirongo icyenda n'icyenda +100000~ibihumbi ijana +200000~ibihumbi magana abiri +101000~ibihumbi ijana na kimwe +130000~ibihumbi ijana na mirongo itatu +531000~ibihumbi magana atanu na mirongo itatu na kimwe +2200345~miliyoni ebyiri n'ibihumbi magana abiri na magana atatu na mirongo ine na gatanu +7000000~miliyoni zirindwi +9101100~miliyoni icyenda n'ibihumbi ijana na kimwe n'ijana +19034004~miliyoni cumi n'icyenda n'ibihumbi mirongo itatu na bine na kane +29000000~miliyoni makumyabiri n'icyenda +40000000~miliyoni mirongo ine +400000000~miliyoni magana ane +100000001~miliyoni ijana na rimwe +340000000~miliyoni magana atatu na mirongo ine +783100000~miliyoni magana arindwi na mirongo inani n'eshatu n'ibihumbi ijana +340010010~miliyoni magana atatu na mirongo ine n'ibihumbi icumi n'icumi +9374514510~tiriyoni icyenda na miliyoni magana atatu na mirongo irindwi n'enye n'ibihumbi magana atanu na cumi na bine na magana atanu n'icumi diff --git a/tests/nemo_text_processing/rw/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..4a4ec27bc --- /dev/null +++ b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_time.txt @@ -0,0 +1,14 @@ +1:00~saa saba +2:01~saa munani n'umunota umwe +3:30~saa cyenda n'iminota mirongo itatu +4:21~saa cumi n'iminota makumyabiri na rimwe +5:12~saa cumi n'imwe n'iminota cumi n'ibiri +6:49~saa cumi n'ebyiri n'iminota mirongo ine n'icyenda +7:05~saa moya n'iminota itanu +8:23~saa mbiri n'iminota makumyabiri n'itatu +9:47~saa tatu n'iminota mirongo ine n'irindwi +10:56~saa ine n'iminota mirongo itanu n'itandatu +11:00~saa tanu +12:09~saa sita n'iminota icyenda +1:59~saa saba n'iminota mirongo itanu n'icyenda +12:31~saa sita n'iminota mirongo itatu n'umwe \ No newline at end of file diff --git a/tests/nemo_text_processing/rw/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..b9b597932 --- /dev/null +++ b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,3 @@ +www~wawawa +maroc~maroki +television~televiziyo \ No newline at end of file diff --git a/tests/nemo_text_processing/rw/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..1c97057aa --- /dev/null +++ b/tests/nemo_text_processing/rw/data_text_normalization/test_cases_word.txt @@ -0,0 +1,26 @@ +~ + ~ + no~no +x ~x +X!~X! +—~— +aabach~aabach +aabenraa~aabenraa +aabye~aabye +aaccessed~aaccessed +aach~aach +aachen's~aachen's +aadri~aadri +aafia~aafia +aagaard~aagaard +aagadu~aagadu +aagard~aagard +aagathadi~aagathadi +aaghart's~aaghart's +aalem~aalem +a'ali~a'ali +aaliyan's~aaliyan's +mother-in-law~mother-in-law +1~rimwe +mar~mar +umwangavu~umwangavu diff --git a/tests/nemo_text_processing/rw/test_cardinal.py b/tests/nemo_text_processing/rw/test_cardinal.py new file mode 100644 index 000000000..eed4be57a --- /dev/null +++ b/tests/nemo_text_processing/rw/test_cardinal.py @@ -0,0 +1,37 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + + normalizer_rw = Normalizer( + input_case='cased', lang='rw', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + @parameterized.expand(parse_test_case_file('rw/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer_rw.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + print(pred) diff --git a/tests/nemo_text_processing/rw/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/rw/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..c67b247e6 --- /dev/null +++ b/tests/nemo_text_processing/rw/test_sparrowhawk_normalization.sh @@ -0,0 +1,60 @@ +#! /bin/sh +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +TEST_DIR=${2:-"/workspace/tests/rw"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read written spoken <<< $testcase + # replace non breaking space with breaking space + # Use below if postprocessor is not used. Comment if it is used + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + # Use below if postprocessor is used. Comment if it is not used + #denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration_pp.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + + + +testTNCardinal() { + input=$TEST_DIR/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + + +testTNTime() { + input=$TEST_DIR/data_text_normalization/test_cases_time.txt + runtest $input +} + + +testTNWhitelist() { + input=$TEST_DIR/data_text_normalization/test_cases_whitelist.txt + runtest $input +} + +testTNWord() { + input=$TEST_DIR/data_text_normalization/test_cases_word.txt + runtest $input +} + + + + + +# Remove all command-line arguments +shift $# + +# Load shUnit2 +. /workspace/shunit2/shunit2 diff --git a/tests/nemo_text_processing/rw/test_time.py b/tests/nemo_text_processing/rw/test_time.py new file mode 100644 index 000000000..a8ada8f73 --- /dev/null +++ b/tests/nemo_text_processing/rw/test_time.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTime: + + normalizer_rw = Normalizer(input_case='cased', lang='rw', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('rw/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer_rw.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/rw/test_whitelist.py b/tests/nemo_text_processing/rw/test_whitelist.py new file mode 100644 index 000000000..3726dbaff --- /dev/null +++ b/tests/nemo_text_processing/rw/test_whitelist.py @@ -0,0 +1,35 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + + normalizer_rw = Normalizer(input_case='cased', lang='rw', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('rw/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer_rw.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/rw/test_word.py b/tests/nemo_text_processing/rw/test_word.py new file mode 100644 index 000000000..10f2e1883 --- /dev/null +++ b/tests/nemo_text_processing/rw/test_word.py @@ -0,0 +1,35 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWord: + + normalizer_rw = Normalizer(input_case='cased', lang='rw', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('rw/data_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer_rw.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_abbreviation.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_abbreviation.txt new file mode 100644 index 000000000..07ccec0a3 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_abbreviation.txt @@ -0,0 +1,5 @@ +ABC~A B C +TV:s~T V:s +PC:in~P C:in +EKG:i~E K G:i +CD:t~C D:t diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt index bfcc14c63..af48fb702 100644 --- a/tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_cardinal.txt @@ -60,3 +60,11 @@ 1 000 000~miljovdna 1.000.000~miljovdna 1 034 068~miljovdnagolbmaloginjealljeduhátguhttalogigávcci +16:t~guhttanuppelogit +16:id~guhttanuppelogiid +16:i~guhttanuppelohkái +16:ide~guhttanuppelogiide +16:s~guhttanuppelogis +16:in~guhttanuppelogiin +16:iguin~guhttanuppelogiiguin +16:n~guhttanuppelohkin diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..2c6255155 --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,10 @@ +1 m~okta mehter +20 m~guoktelogi mehtera +22 m~guoktelogiguokte mehtera +20 km/d~guoktelogi kilomehtera diimmus +12 m/s~guoktenuppelohkái mehtera sekunddas +5-jahkásaš~viđajahkásaš +9-jahkásaččat~ovccejahkáččat +12-jahkásaččat~guoktenuppelohjahkáččat +20-jahkásaš~guoktelogijahkásaš +22-jahkásaš~guoktelogiguovttejahkásaš diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..c9e703ebb --- /dev/null +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_money.txt @@ -0,0 +1,8 @@ +1 kr~okta ruvdno +20 kr~guoktelogi ruvnno +50 NOK~vihttalogi norgga ruvnno +40 SEK~njealljelogi ruoŧa ruvnno +25 $~guoktelogivihtta dollára +14 £~njealljenuppelohkái punda +40 €~njealljelogi euro +99,– kr~ovccilogiovcci ruvnno diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt index ec6d620c3..bd17237b4 100644 --- a/tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_ordinal.txt @@ -36,3 +36,12 @@ 90.~ovccilogát 100.~čuođát 1000.~duháhat +16:dat~guhttanuppelogádat +16:da~guhttanuppelogáda +16:diid~guhttanuppelogádiid +16:dii~guhttanuppelogádii +16:diidda~guhttanuppelogádiidda +16:dis~guhttanuppelogádis +16:diin~guhttanuppelogádiin +16:diiguin~guhttanuppelogádiiguin +16:din~guhttanuppelogádin diff --git a/tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt index 9dc1f9641..51080725a 100644 --- a/tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt +++ b/tests/nemo_text_processing/se/data_text_normalization/test_cases_whitelist.txt @@ -1 +1,6 @@ tel.~telefonnummár +abstr.~abstrákta +dii.~diibmu +gč.~geahča +tlf.~telefovdna +vrd.~veardit diff --git a/tests/nemo_text_processing/se/test_abbreviation.py b/tests/nemo_text_processing/se/test_abbreviation.py new file mode 100644 index 000000000..ef21721fd --- /dev/null +++ b/tests/nemo_text_processing/se/test_abbreviation.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, Jim O'Regan. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib.rewrite import top_rewrite + +from nemo_text_processing.text_normalization.se.taggers.abbreviation import AbbreviationFst + +from ..utils import parse_test_case_file + + +class TestAbbreviation: + abbreviation = AbbreviationFst(whitelist=None, deterministic=False) + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_abbreviation.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + assert top_rewrite(test_input, self.abbreviation.graph) == expected diff --git a/tests/nemo_text_processing/se/test_cardinal.py b/tests/nemo_text_processing/se/test_cardinal.py index 11cd08d26..43034eeb1 100644 --- a/tests/nemo_text_processing/se/test_cardinal.py +++ b/tests/nemo_text_processing/se/test_cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/se/test_date.py b/tests/nemo_text_processing/se/test_date.py index 745a45d2c..2633208d2 100644 --- a/tests/nemo_text_processing/se/test_date.py +++ b/tests/nemo_text_processing/se/test_date.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/se/test_electronic.py b/tests/nemo_text_processing/se/test_electronic.py index 1d0ea03e8..fa6fefd77 100644 --- a/tests/nemo_text_processing/se/test_electronic.py +++ b/tests/nemo_text_processing/se/test_electronic.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/se/test_measure.py b/tests/nemo_text_processing/se/test_measure.py new file mode 100644 index 000000000..24ac40f9c --- /dev/null +++ b/tests/nemo_text_processing/se/test_measure.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, Jim O'Regan. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib.rewrite import top_rewrite + +from nemo_text_processing.text_normalization.se.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.se.taggers.measure import MeasureFst + +from ..utils import parse_test_case_file + + +class TestMeasure: + graph = MeasureFst(CardinalFst(deterministic=True), deterministic=True).graph + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + assert top_rewrite(test_input, self.graph) == expected diff --git a/tests/nemo_text_processing/se/test_money.py b/tests/nemo_text_processing/se/test_money.py new file mode 100644 index 000000000..0b6bec6af --- /dev/null +++ b/tests/nemo_text_processing/se/test_money.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, Jim O'Regan. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from pynini.lib.rewrite import top_rewrite + +from nemo_text_processing.text_normalization.se.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.se.taggers.money import MoneyFst + +from ..utils import parse_test_case_file + + +class TestMoney: + graph = MoneyFst(CardinalFst(deterministic=True), deterministic=True).graph + + @parameterized.expand(parse_test_case_file('se/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + assert top_rewrite(test_input, self.graph) == expected diff --git a/tests/nemo_text_processing/se/test_ordinal.py b/tests/nemo_text_processing/se/test_ordinal.py index 2008c8201..01bec46e5 100644 --- a/tests/nemo_text_processing/se/test_ordinal.py +++ b/tests/nemo_text_processing/se/test_ordinal.py @@ -14,9 +14,10 @@ import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/se/test_time.py b/tests/nemo_text_processing/se/test_time.py index ae43cec51..05ca1871d 100644 --- a/tests/nemo_text_processing/se/test_time.py +++ b/tests/nemo_text_processing/se/test_time.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/se/test_whitelist.py b/tests/nemo_text_processing/se/test_whitelist.py index bf208fd1b..4e7c1bb56 100644 --- a/tests/nemo_text_processing/se/test_whitelist.py +++ b/tests/nemo_text_processing/se/test_whitelist.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/se/test_word.py b/tests/nemo_text_processing/se/test_word.py index 2193487e2..d2878f928 100644 --- a/tests/nemo_text_processing/se/test_word.py +++ b/tests/nemo_text_processing/se/test_word.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_cardinal.py b/tests/nemo_text_processing/sv/test_cardinal.py index 99ca5b48b..7f0914ec2 100644 --- a/tests/nemo_text_processing/sv/test_cardinal.py +++ b/tests/nemo_text_processing/sv/test_cardinal.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_date.py b/tests/nemo_text_processing/sv/test_date.py index 495150df0..b0a53e46c 100644 --- a/tests/nemo_text_processing/sv/test_date.py +++ b/tests/nemo_text_processing/sv/test_date.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_decimal.py b/tests/nemo_text_processing/sv/test_decimal.py index e1d6a6b49..034e3d4d8 100644 --- a/tests/nemo_text_processing/sv/test_decimal.py +++ b/tests/nemo_text_processing/sv/test_decimal.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_electronic.py b/tests/nemo_text_processing/sv/test_electronic.py index e9492a063..b2a828847 100644 --- a/tests/nemo_text_processing/sv/test_electronic.py +++ b/tests/nemo_text_processing/sv/test_electronic.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_fraction.py b/tests/nemo_text_processing/sv/test_fraction.py index f807a4179..67253f272 100644 --- a/tests/nemo_text_processing/sv/test_fraction.py +++ b/tests/nemo_text_processing/sv/test_fraction.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_measure.py b/tests/nemo_text_processing/sv/test_measure.py index 185d407d2..a4b026bbd 100644 --- a/tests/nemo_text_processing/sv/test_measure.py +++ b/tests/nemo_text_processing/sv/test_measure.py @@ -14,9 +14,10 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_money.py b/tests/nemo_text_processing/sv/test_money.py index e9e875b99..b5b065b48 100644 --- a/tests/nemo_text_processing/sv/test_money.py +++ b/tests/nemo_text_processing/sv/test_money.py @@ -14,9 +14,10 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_normalization_with_audio.py b/tests/nemo_text_processing/sv/test_normalization_with_audio.py index 105a60c60..bd9e88cbc 100644 --- a/tests/nemo_text_processing/sv/test_normalization_with_audio.py +++ b/tests/nemo_text_processing/sv/test_normalization_with_audio.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, get_test_cases_multiple diff --git a/tests/nemo_text_processing/sv/test_ordinal.py b/tests/nemo_text_processing/sv/test_ordinal.py index ffc6922f0..6cd11a4b0 100644 --- a/tests/nemo_text_processing/sv/test_ordinal.py +++ b/tests/nemo_text_processing/sv/test_ordinal.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/sv/test_sparrowhawk_inverse_text_normalization.sh index 2ed171ccf..350136c4a 100644 --- a/tests/nemo_text_processing/sv/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/sv/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/sv/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/sv/test_sparrowhawk_normalization.sh index 71a6466fb..49480ee6a 100644 --- a/tests/nemo_text_processing/sv/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/sv/test_sparrowhawk_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/sv/test_telephone.py b/tests/nemo_text_processing/sv/test_telephone.py index 3e806ebd9..40e5c846f 100644 --- a/tests/nemo_text_processing/sv/test_telephone.py +++ b/tests/nemo_text_processing/sv/test_telephone.py @@ -14,10 +14,11 @@ import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_time.py b/tests/nemo_text_processing/sv/test_time.py index a281b18e3..feaa637f2 100644 --- a/tests/nemo_text_processing/sv/test_time.py +++ b/tests/nemo_text_processing/sv/test_time.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_whitelist.py b/tests/nemo_text_processing/sv/test_whitelist.py index 176744a87..5fba8e99a 100644 --- a/tests/nemo_text_processing/sv/test_whitelist.py +++ b/tests/nemo_text_processing/sv/test_whitelist.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/sv/test_word.py b/tests/nemo_text_processing/sv/test_word.py index 636e7fe3a..c92f148f5 100644 --- a/tests/nemo_text_processing/sv/test_word.py +++ b/tests/nemo_text_processing/sv/test_word.py @@ -13,10 +13,11 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -from parameterized import parameterized from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file diff --git a/tests/nemo_text_processing/utils.py b/tests/nemo_text_processing/utils.py index 5326784e9..468be7300 100644 --- a/tests/nemo_text_processing/utils.py +++ b/tests/nemo_text_processing/utils.py @@ -39,7 +39,7 @@ def parse_test_case_file(file_name: str): Prepares tests pairs for ITN and TN tests """ test_pairs = [] - with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f: + with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r', encoding='utf-8') as f: for line in f: components = line.strip("\n").split("~") spoken = components[0] @@ -63,7 +63,7 @@ def get_test_cases_multiple(file_name: str = 'data_text_normalization/en/test_ca Prepares tests pairs for audio based TN tests """ test_pairs = [] - with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f: + with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r', encoding='utf-8') as f: written = None normalized_options = [] for line in f: diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_cardinal.txt index 53853297c..ca1f1a0aa 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_cardinal.txt @@ -90,4 +90,53 @@ một trăm một~100 một một trăm linh một~101 một trăm mốt~110 một trăm mười~110 -hai triệu ba nghìn~2003000 \ No newline at end of file +hai triệu ba nghìn~2003000 +mười một~11 +mười hai~12 +mười lăm~15 +mười bảy~17 +hai mươi~20 +hai mươi mốt~21 +hai mốt~21 +ba mươi lăm~35 +bốn mươi tư~44 +năm mươi lăm~55 +bảy mươi mốt~71 +chín mươi chín~99 +một trăm~100 +một trăm linh một~101 +một trăm lẻ năm~105 +hai trăm mười hai~212 +ba trăm linh tư~304 +năm trăm linh lăm~505 +sáu trăm lẻ tư~604 +bảy trăm lẻ bảy~707 +chín trăm linh chín~909 +chín trăm chín mươi chín~999 +một nghìn~1000 +một ngàn~1000 +một nghìn không trăm linh năm~1005 +một ngàn hai trăm ba mươi bốn~1234 +mười hai nghìn ba trăm bốn mươi lăm~12345 +một trăm hai mươi ba nghìn không trăm linh bốn~123004 +một triệu~1 triệu +hai triệu ba trăm nghìn bốn trăm lẻ hai~2300402 +hai triệu ba trăm nghìn không trăm linh bảy~2300007 +một tỷ~1 tỷ +ba tỷ không trăm linh ba~3000000003 +ba tỷ bốn trăm năm mươi sáu triệu bảy trăm tám mươi chín nghìn không trăm linh một~3456789001 +một nghìn tỷ~1000 tỷ +một triệu tỷ~1 triệu tỷ +một tỷ tỷ~1 tỷ tỷ +hai mươi ba tỷ bốn trăm năm mươi sáu triệu bảy trăm tám mươi chín nghìn không trăm mười hai~23456789012 +chín trăm chín mươi chín tỷ chín trăm chín mươi chín triệu chín trăm chín mươi chín nghìn chín trăm chín mươi chín~999999999999 +một trăm hai mươi ba tỷ bốn trăm năm mươi sáu triệu bảy trăm tám mươi chín nghìn không trăm mười hai~123456789012 +âm bốn mươi hai~-42 +âm một trăm lẻ tám~-108 +âm ba trăm nghìn không trăm linh năm~-300005 +âm ba triệu không trăm linh chín~-3000009 + ba mươi mốt ~31 +hai mươi mốt~21 +bốn ngàn ba trăm lẻ năm~4305 +iPhone mười lăm~iPhone 15 +đường số mười hai~đường số 12 \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_date.txt index 182c710e2..21fb26138 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_date.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_date.txt @@ -13,4 +13,57 @@ năm chín trăm ba tám~năm 938 năm ba trăm lẻ tám~năm 308 năm bẩy trăm bốn mươi tư~năm 744 học kỳ này sẽ kết thúc vào tháng tư ngày mười tháng năm là tổng kết~học kỳ này sẽ kết thúc vào tháng 4 ngày 10 tháng 5 là tổng kết -học kỳ này sẽ kết thúc vào tháng năm ngày một tháng sáu là tổng kết~học kỳ này sẽ kết thúc vào tháng năm ngày 1 tháng 6 là tổng kết \ No newline at end of file +học kỳ này sẽ kết thúc vào tháng năm ngày một tháng sáu là tổng kết~học kỳ này sẽ kết thúc vào tháng năm ngày 1 tháng 6 là tổng kết +mùng một tháng một năm một chín chín chín~mùng 1 tháng 1 năm 1999 +ngày mùng năm tháng tám~ngày mùng 5 tháng 8 +mồng hai tháng ba~mồng 2 tháng 3 +ngày mồng mười tháng chín~ngày mồng 10 tháng 9 +hai mươi tư tháng bảy~24 tháng 7 +ngày ba mươi tháng sáu~ngày 30 tháng 6 +tháng một~tháng 1 +tháng hai~tháng 2 +tháng ba~tháng 3 +tháng tư~tháng 4 +tháng năm~tháng năm +tháng sáu~tháng 6 +tháng bảy~tháng 7 +tháng tám~tháng 8 +tháng chín~tháng 9 +tháng mười~tháng 10 +tháng mười một~tháng 11 +tháng mười hai~tháng 12 +tháng mười năm hai ngàn linh chín~tháng 10 năm 2009 +năm chín trăm ba tám~năm 938 +năm ba trăm lẻ tám~năm 308 +năm bẩy trăm bốn mươi tư~năm 744 +năm một chín tám tư~năm 1984 +năm hai nghìn linh chín~năm 2009 +năm hai không linh chín~năm 2009 +năm hai không hai ba~năm 2023 +năm hai nghìn không trăm hai mươi bốn~năm 2024 +tháng ba năm một trăm linh năm~tháng 3 năm 105 +tháng tư năm ba trăm lẻ tám~tháng 4 năm 308 +tháng tám năm chín trăm ba tám~tháng 8 năm 938 +tháng năm năm một nghìn không trăm linh ba~tháng 5 năm 1003 +tháng mười một năm một nghìn tám trăm năm hai~tháng 11 năm 1852 +tháng bảy năm một nghìn chín trăm tám tư~tháng 7 năm 1984 +tháng tư năm một chín bảy lăm~tháng 4 năm 1975 +tháng mười hai năm hai nghìn linh hai~tháng 12 năm 2002 +tháng sáu năm hai nghìn không trăm lẻ năm~tháng 6 năm 2005 +tháng một năm hai nghìn mười~tháng 1 năm 2010 +tháng mười năm hai nghìn mười chín~tháng 10 năm 2019 +tháng hai năm hai ngàn mười ba~tháng 2 năm 2013 +tháng chín năm hai không không tám~tháng 9 năm 2008 +tháng mười năm hai không một sáu~tháng 10 năm 2016 +tháng ba năm hai không hai ba~tháng 3 năm 2023 +tháng bảy năm hai không hai bốn~tháng 7 năm 2024 +tháng tư năm hai nghìn không trăm linh chín~tháng 4 năm 2009 +tháng tám năm hai ngàn không trăm mười bảy~tháng 8 năm 2017 +tháng mười một năm hai nghìn không trăm hai mốt~tháng 11 năm 2021 +tháng năm năm hai ngàn hai mươi hai~tháng 5 năm 2022 +tháng tư năm hai không mười tám~tháng 4 năm 2018 +tháng bảy năm hai không mười chín~tháng 7 năm 2019 +tháng ba năm hai không linh sáu~tháng 3 năm 2006 +tháng mười một năm hai không linh tám~tháng 11 năm 2008 +tháng sáu năm hai không hai mốt~tháng 6 năm 2021 +tháng mười hai năm hai không hai hai~tháng 12 năm 2022 \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_decimal.txt index 9888ff64e..654c66e61 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_decimal.txt @@ -22,4 +22,53 @@ trừ chín chín tỷ chín~-99.9 tỷ âm chín chín chấm chín lăm tỷ~-99.95 tỷ hai mươi chấm tư~20.4 mười hai chấm mốt~12 chấm mốt -chín trăm chín ba chấm lăm~993 chấm lăm \ No newline at end of file +chín trăm chín ba chấm lăm~993 chấm lăm +không phẩy năm~0.5 +không phẩy không năm~0.05 +không phẩy không không một~0.001 +một phẩy năm~1.5 +một phẩy không năm~1.05 +hai phẩy ba~2.3 +hai phẩy không ba~2.03 +mười hai phẩy ba bốn~12.34 +một trăm phẩy không không một~100.001 +một nghìn phẩy không năm~1000.05 +một triệu phẩy không không không một~1000000.0001 +một phẩy hai ba không năm~1.2305 +hai mươi ba phẩy không không bốn~23.004 +ba tỷ phẩy không không không không một~3000000000.00001 +một chấm năm~1.5 +không chấm không không một~0.001 +âm không phẩy năm~-0.5 +âm mười hai phẩy không hai~-12.02 +âm một trăm lẻ ba phẩy bốn năm sáu~-103.456 +một tỷ rưỡi~1.5 tỷ +một trăm hai mươi ba tỷ bốn trăm năm mươi sáu triệu phẩy bảy tám chín~123456000000.789 +một nghìn không trăm linh năm phẩy không hai~1005.02 + ba phẩy không bảy ~3.07 +một nghìn linh một chấm không năm~1001.05 +hai nghìn lẻ ba phẩy bảy tám~2003.78 +ba triệu linh một phẩy không không chín~3000001.009 +bốn tỷ lẻ năm chấm hai ba bốn~4000000005.234 +năm triệu không trăm mười phẩy năm sáu~5000010.56 +một tỷ hai trăm ba mươi bốn triệu năm trăm sáu mươi bảy nghìn tám trăm chín mươi chấm một hai ba~1234567890.123 +chín tỷ tám trăm bảy mươi sáu triệu năm trăm bốn mươi ba nghìn hai trăm mười phẩy chín tám bảy sáu~9876543210.9876 +hai mươi ba tỷ bốn mươi lăm triệu sáu trăm bảy mươi tám nghìn chín trăm phẩy một hai ba bốn năm~23045678900.12345 +hai mươi mốt chấm tư~21.4 +ba mươi tư phẩy năm~34.5 +năm mươi mốt phẩy bảy~51.7 +sáu mươi tư chấm tám chín~64.89 +không phẩy không không không một~0.0001 +không chấm không không không không một~0.00001 +một phẩy một hai ba bốn năm sáu bảy tám chín~1.123456789 +hai chấm không một hai ba bốn năm sáu bảy~2.01234567 +âm mười hai tỷ ba trăm bốn mươi lăm triệu sáu trăm bảy mươi tám nghìn chín trăm phẩy một hai~-12345678900.12 +trừ bảy trăm tám mươi chín triệu một trăm hai mươi ba nghìn bốn trăm năm mươi sáu chấm bảy tám~-789123456.78 +âm chín trăm chín mươi chín triệu chín trăm chín mươi chín nghìn chín trăm chín mươi chín phẩy chín chín chín~-999999999.999 +một tỷ rưỡi~1.5 tỷ +ba triệu mốt~3.1 triệu +chín mươi chín phẩy chín chín~99.99 +một trăm phẩy không~100.0 +một chấm năm triệu~1.5 triệu +ba phẩy bốn nghìn~3.4 nghìn +một trăm hai mươi ba phẩy bốn năm sáu bảy tám chín~123.456789 \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_electronic.txt index 04168797e..3d0767cdf 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_electronic.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_electronic.txt @@ -7,4 +7,12 @@ a b c a móc a b c chấm com~abc@abc.com a s d f một hai ba a móc a b c chấm com~asdf123@abc.com a một b hai a vòng a b c chấm com~a1b2@abc.com a b ba chấm s d d chấm ba a móc g mail chấm com~ab3.sdd.3@gmail.com -a b ba chấm s d d chấm ba a còng g mail chấm com~ab3.sdd.3@gmail.com \ No newline at end of file +a b ba chấm s d d chấm ba a còng g mail chấm com~ab3.sdd.3@gmail.com +a b ba gạch s d d gạch ba a móc g mail chấm com~ab3-sdd-3@gmail.com +w w w chấm nvidia chấm com~www.nvidia.com +nvidia chấm com~nvidia.com +h t t p hai chấm sẹc sẹc w w w chấm nvidia chấm com~http://www.nvidia.com +h t t p s hai chấm sẹc sẹc w w w chấm nvidia chấm com~https://www.nvidia.com +google chấm com chấm v n~google.com.vn +w w w chấm google chấm com chấm v n~www.google.com.vn +nvidia chấm a i~nvidia.ai \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_measure.txt index 8b8c97b96..48d0577b6 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_measure.txt @@ -14,9 +14,22 @@ bốn trăm bốn mươi mi li lít~440 ml ba trăm muy crô mét~300 μm sáu lăm inch~65 inch hai vôn~2 v -ba mươi phần trăm~30 % +ba mươi phần trăm~30% sáu mươi nghìn hai trăm bốn mươi mi li ampe~60240 mA sáu mươi sáu phút~66 phút hai phút~2 phút -năm giây~5 s -năm trăm sáu bảy giây~567 s \ No newline at end of file +năm giây~5s +năm trăm sáu bảy giây~567s +hai mươi độ c~20 °c +chín mươi tám độ f~98 °f +ba bảy độ c~37 °c +không phẩy năm phần trăm~0.5% +một trăm phần trăm~100% +không phẩy không năm mét~0.05 m +không phẩy không không một ki lô gram~0.001 kg +hai trăm megabit trên giây~200 mbps +năm mươi kilobit trên giây~50 kbps +một phẩy năm terabyte~1.5 tb +hai phẩy năm gigabyte~2.5 gb +không mét~0 m +không phần trăm~0% \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_money.txt index 2d99fd4bb..064ebae56 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_money.txt @@ -6,12 +6,10 @@ hai phẩy hai đô la mỹ~2.2$ hai đô rưỡi~2.5$ hai mươi euro~20€ hai mươi chấm một euro~20.1€ -một đồng~1₫ -mười nghìn năm trăm đồng~10500₫ -năm phẩy sáu đồng~5.6₫ -hai mươi đồng rưỡi~20.5₫ tám mươi nghìn một trăm won~80100₩ tám mươi ngàn một trăm uôn~80100₩ ba ringgit~3RM không phẩy ba ringgit~0.3RM -không euro~0€ \ No newline at end of file +không euro~0€ +không phẩy không không một đô~0.001$ +một phẩy hai ba bốn đô la~1.234$ \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_telephone.txt index 0fa73bcaa..7d8e80613 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_telephone.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_telephone.txt @@ -1,4 +1,25 @@ -không chín ba sáu năm năm năm bốn bốn chín~0936555449 -không một hai tám bốn hai hai năm~01284225 +không chín ba sáu năm năm năm bốn bốn chín~093-655-5449 +không ba tám bốn hai hai năm bảy tám chín~038-422-5789 +không bảy chín một hai ba bốn năm sáu bảy~079-123-4567 +không hai bốn ba bảy hai một năm sáu tám chín~02437215689 +không hai tám ba tám hai một năm sáu tám chín~02838215689 +không một hai tám bốn hai hai năm bảy tám chín~01284225789 một hai ba bốn năm sáu bảy tám chín~123456789 -chín tám bảy sáu năm bốn ba hai một không~9876543210 \ No newline at end of file +một hai ba bốn năm sáu bảy tám mốt~123456781 +chín tám bảy sáu năm bốn ba hai một không~9876543210 +một một hai~112 +một một ba~113 +một một bốn~114 +một một năm~115 +một chín không năm~1905 +cộng tám mươi bốn không chín ba sáu năm năm năm bốn bốn chín~+84 093-655-5449 +cộng một không chín ba sáu năm năm năm bốn bốn chín~+1 093-655-5449 +cộng chín một không ba tám bốn hai hai năm bảy tám chín~+91 038-422-5789 +cộng bốn bốn không bảy chín một hai ba bốn năm sáu bảy~+44 079-123-4567 +một chín hai chấm một sáu tám chấm không chấm một~192.168.0.1 +một chín hai chấm một sáu tám chấm một chấm một~192.168.1.1 +hai năm năm chấm hai năm năm chấm hai năm năm chấm hai năm năm~255.255.255.255 +một không chấm không chấm không chấm không~10.0.0.0 +một hai ba bốn năm sáu bảy tám chín không một hai ba bốn năm sáu~1234 5678 9012 3456 +bốn năm ba hai một hai ba bốn năm sáu bảy tám chín không một hai~4532 1234 5678 9012 +ba bảy năm ba bốn hai sáu hai bốn hai sáu hai bốn hai sáu~3753 426242 62426 \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_time.txt index 4718c6d6c..aad0635d6 100644 --- a/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/vi/data_inverse_text_normalization/test_cases_time.txt @@ -19,4 +19,13 @@ mười hai phút ba giây~12p03s năm chín phút năm mươi chín giây~59p59s tám phút bốn lăm giây~8p45s tám giờ hai ba phút gmt~8:23 gmt -mười lăm giờ cst~15h cst \ No newline at end of file +mười lăm giờ cst~15h cst +năm giờ linh năm~5:05 +sáu giờ không tám phút~6:08 +ba giờ mười giây~3h 10s +bốn giờ kém mười~3:50 +không giờ không phút~0:00 +hai giờ không một phút~2:01 +mười một giờ kém năm~10:55 +mười một giờ kém mười~10:50 +một giờ chiều~1h chiều \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..74d2b7e98 --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,109 @@ +1~một +2~hai +3~ba +4~bốn +5~năm +6~sáu +7~bảy +8~tám +9~chín +10~mười +11~mười một +12~mười hai +15~mười lăm +18~mười tám +19~mười chín +20~hai mươi +21~hai mươi mốt +25~hai mươi lăm +30~ba mươi +34~ba mươi tư +44~bốn mươi tư +55~năm mươi lăm +67~sáu mươi bảy +70~bảy mươi +80~tám mươi +95~chín mươi lăm +100~một trăm +101~một trăm linh một +102~một trăm linh hai +104~một trăm linh bốn +105~một trăm linh năm +110~một trăm mười +111~một trăm mười một +120~một trăm hai mươi +123~một trăm hai mươi ba +200~hai trăm +201~hai trăm linh một +500~năm trăm +999~chín trăm chín mươi chín +1000~một nghìn +1001~một nghìn linh một +1020~một nghìn hai mươi +1095~một nghìn chín mươi lăm +1100~một nghìn một trăm +2000~hai nghìn +10000~mười nghìn +100000~một trăm nghìn +1000000~một triệu +2000000~hai triệu +1000000000~một tỷ +-1~âm một +-25~âm hai mươi lăm +-100~âm một trăm +-1000~âm một nghìn +0~không +24~hai mươi tư +35~ba mươi lăm +41~bốn mươi mốt +55~năm mươi lăm +91~chín mươi mốt +14~mười bốn +16~mười sáu +17~mười bảy +37~ba mươi bảy +47~bốn mươi bảy +57~năm mươi bảy +63~sáu mươi ba +79~bảy mươi chín +84~tám mươi tư +98~chín mươi tám +-123~âm một trăm hai mươi ba +-1001~âm một nghìn linh một +-104~âm một trăm linh bốn +1000001~một triệu linh một +1001001~một triệu một nghìn linh một +1050003~một triệu năm mươi nghìn linh ba +1000000001~một tỷ linh một +1001001101~một tỷ một triệu một nghìn một trăm linh một +300~ba trăm +400~bốn trăm +500~năm trăm +6000~sáu nghìn +7000~bảy nghìn +15000~mười lăm nghìn +300000~ba trăm nghìn +450000~bốn trăm năm mươi nghìn +5000000~năm triệu +700000000~bảy trăm triệu +31~ba mươi mốt +41~bốn mươi mốt +51~năm mươi mốt +61~sáu mươi mốt +71~bảy mươi mốt +81~tám mươi mốt +91~chín mươi mốt +5500000~năm triệu năm trăm nghìn +1000010~một triệu mười +1000100~một triệu một trăm +1000101~một triệu một trăm linh một +1010001~một triệu mười nghìn linh một +10000000000~mười tỷ +150~một trăm năm mươi +1000000000000~một nghìn tỷ +1234567890123~một nghìn hai trăm ba mươi tư tỷ năm trăm sáu mươi bảy triệu tám trăm chín mươi nghìn một trăm hai mươi ba +9876543210987~chín nghìn tám trăm bảy mươi sáu tỷ năm trăm bốn mươi ba triệu hai trăm mười nghìn chín trăm tám mươi bảy +1000000000000000~một triệu tỷ +1111111111111111~một triệu một trăm mười một nghìn một trăm mười một tỷ một trăm mười một triệu một trăm mười một nghìn một trăm mười một +5432109876543210~năm triệu bốn trăm ba mươi hai nghìn một trăm linh chín tỷ tám trăm bảy mươi sáu triệu năm trăm bốn mươi ba nghìn hai trăm mười +1000000000000000000~một tỷ tỷ \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_date.txt new file mode 100644 index 000000000..c95e00e97 --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_date.txt @@ -0,0 +1,13 @@ +ngày 15/01/2024~ngày mười lăm tháng một năm hai nghìn hai mươi tư +01/12/2023~ngày một tháng mười hai năm hai nghìn hai mươi ba +25-03-1975~ngày hai mươi lăm tháng ba năm một nghìn chín trăm bảy mươi lăm +10.05.2000~ngày mười tháng năm năm hai nghìn +tháng 1 2024~tháng một năm hai nghìn hai mươi tư +tháng 12 2023~tháng mười hai năm hai nghìn hai mươi ba +ngày 12 tháng 5 năm 2025~ngày mười hai tháng năm năm hai nghìn hai mươi lăm +tháng 5 năm nay~tháng năm năm nay +ngày 4 tháng này~ngày bốn tháng này +hôm nay là ngày 19/05/2025 sinh nhật Bác Hồ~hôm nay là ngày mười chín tháng năm năm hai nghìn hai mươi lăm sinh nhật Bác Hồ +ngày 14/4 hàng năm~ngày mười bốn tháng tư hàng năm +tháng 04/1969~tháng tư năm một nghìn chín trăm sáu mươi chín +ngày 12 tháng mười hai năm 2023~ngày mười hai tháng mười hai năm hai nghìn hai mươi ba \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_decimal.txt new file mode 100644 index 000000000..6acc3bda4 --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_decimal.txt @@ -0,0 +1,29 @@ +0,2 triệu~không phẩy hai triệu +18 vạn~mười tám vạn +818,303~tám trăm mười tám phẩy ba không ba +-99,95 tỷ~âm chín mươi chín phẩy chín năm tỷ +60,240~sáu mươi phẩy hai bốn không +-0,007~âm không phẩy không không bảy +123,000~một trăm hai mươi ba phẩy không không không +1,5 triệu~một phẩy năm triệu +3,14 tỷ~ba phẩy một bốn tỷ +10,01 vạn~mười phẩy không một vạn +-12,5~âm mười hai phẩy năm +0,0001~không phẩy không không không một +999,999~chín trăm chín mươi chín phẩy chín chín chín +1,01~một phẩy không một +-1,01~âm một phẩy không một +15,6~mười lăm phẩy sáu +1k~một nghìn +10k~mười nghìn +100k~một trăm nghìn +1tr~một triệu +10tr~mười triệu +100tr~một trăm triệu +1tr2~một triệu hai trăm nghìn +2tr5~hai triệu năm trăm nghìn +1t~một tỷ +10t~mười tỷ +100t~một trăm tỷ +2t3~hai tỷ ba trăm triệu +1 tỉ~một tỉ \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_fraction.txt new file mode 100644 index 000000000..1ccd7af94 --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_fraction.txt @@ -0,0 +1,13 @@ +1/2~một phần hai +4/9~bốn phần chín +9/4~chín phần tư +1/4~một phần tư +3/4~ba phần tư +15/5~mười lăm phần năm +1/3~một phần ba +2/10~hai phần mười +23 1/5~hai mươi ba và một phần năm +-3/4~âm ba phần tư +-12 1/4 nha~âm mười hai và một phần tư nha +-5 2/3~âm năm và hai phần ba +5 1/2~năm và một phần hai \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..d3a7adeaa --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,63 @@ +204m~hai trăm linh bốn mét +12kg~mười hai ki lô gam +1kg~một ki lô gam +100g~một trăm gam +500mg~năm trăm mi li gam +175cm~một trăm bảy mươi lăm xăng ti mét +2m~hai mét +100mm~một trăm mi li mét +5km~năm ki lô mét +1inch~một inch +500ml~năm trăm mi li lít +2l~hai lít +1m³~một mét khối +100cm³~một trăm xăng ti mét khối +2gb~hai gi ga bai +1tb~một terabyte +512Mb~năm trăm mười hai mê ga bai +64kb~sáu mươi tư ki lô bai +25°c~hai mươi lăm độ c +100°f~một trăm độ f +273°k~hai trăm bảy mươi ba độ k +50%~năm mươi phần trăm +100%~một trăm phần trăm +25%~hai mươi lăm phần trăm +220v~hai trăm hai mươi vôn +1kw~một ki lô oát +500mV~năm trăm mi li vôn +1000mA~một nghìn mi li am pe +50hz~năm mươi hẹc +2ghz~hai gi ga hẹc +100Mhz~một trăm mê ga hẹc +1000kw~một nghìn ki lô oát +5hp~năm mã lực +1tw~một tê ra oát +100m²~một trăm mét vuông +5km²~năm ki lô mét vuông +1km2~một ki lô mét vuông +8,5m2~tám phẩy năm mét vuông +1ha~một héc ta +1/2kg~một phần hai ki lô gam +3/4m~ba phần tư mét +1/3l~một phần ba lít +Tôi có 12kg gạo~Tôi có mười hai ki lô gam gạo +Chiều cao 175cm~Chiều cao một trăm bảy mươi lăm xăng ti mét +Dung lượng 2gb~Dung lượng hai gi ga bai +Nhiệt độ 25°c~Nhiệt độ hai mươi lăm độ c +Cân nặng 1/2kg~Cân nặng một phần hai ki lô gam +Điện áp 220v~Điện áp hai trăm hai mươi vôn +Tỷ lệ 50%~Tỷ lệ năm mươi phần trăm +Bộ nhớ 1tb~Bộ nhớ một terabyte +Thể tích 500ml~Thể tích năm trăm mi li lít +1234kg~một nghìn hai trăm ba mươi tư ki lô gam +2500m~hai nghìn năm trăm mét +10000gb~mười nghìn gi ga bai +Kích thước 100cm x 50cm~Kích thước một trăm xăng ti mét x năm mươi xăng ti mét +1,5m2~một phẩy năm mét vuông +1,5m~một phẩy năm mét +120km/h~một trăm hai mươi ki lô mét trên giờ +100 km/h~một trăm ki lô mét trên giờ +50m/s~năm mươi mét trên giây +30 m/min~ba mươi mét trên phút +5cm/s~năm xăng ti mét trên giây +200mg/ml~hai trăm mi li gam trên mi li lít \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..755a1030a --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_money.txt @@ -0,0 +1,30 @@ +10₫~mười đồng +10 đồng~mười đồng +10,5 đồng~mười phẩy năm đồng +10$~mười đô la +1000$~một nghìn đô la +10 đô la~mười đô la +10 usd~mười đô la +10,5$~mười đô la năm mươi xu +10€~mười ơ rô +10¥~mười yên nhật +10£~mười bảng anh +10₹~mười rupee +0,5$~năm mươi xu +12.345 đồng~mười hai nghìn ba trăm bốn mươi lăm đồng +1.234.567₫~một triệu hai trăm ba mươi tư nghìn năm trăm sáu mươi bảy đồng +10.000,50 đồng~mười nghìn phẩy năm không đồng +10k VND~mười nghìn đồng +10đ~mười đồng +50.000đ-100.000đ~năm mươi nghìn đồng đến một trăm nghìn đồng +10$-20$~mười đô la đến hai mươi đô la +50.000đ/ngày~năm mươi nghìn đồng trên ngày +10$/giờ~mười đô la trên giờ +5€/h~năm ơ rô trên giờ +100₫/kg~một trăm đồng một ki lô gam +1tr5 vnd~một triệu năm trăm nghìn đồng +0,01$~một xu +2,50€~hai ơ rô năm mươi xu +1000,50 VND~một nghìn phẩy năm không đồng +5,99$~năm đô la chín mươi chín xu +30đ/TB~ba mươi đồng một tê ra bai \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_ordinal.txt new file mode 100644 index 000000000..e28b4f97f --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_ordinal.txt @@ -0,0 +1,30 @@ +thứ 1~thứ nhất +hôm nay là thứ hai~hôm nay là thứ hai +thứ 3 là ngày giữa tuần~thứ ba là ngày giữa tuần +thứ 4 nên làm gì~thứ tư nên làm gì +thứ 7~thứ bảy +con giáp thứ 13~con giáp thứ mười ba +thứ 1~thứ nhất +thứ 4~thứ tư +thứ 2~thứ hai +thứ 3~thứ ba +thứ 5~thứ năm +thứ 6~thứ sáu +thứ 7~thứ bảy +thứ 8~thứ tám +thứ 9~thứ chín +thứ 10~thứ mười +thứ 11~thứ mười một +thứ 12~thứ mười hai +thứ 15~thứ mười lăm +thứ 21~thứ hai mươi mốt +thứ 24~thứ hai mươi tư +thứ 34~thứ ba mươi tư +thứ 100~thứ một trăm +thứ 101~thứ một trăm linh một +thứ 104~thứ một trăm linh bốn +thứ 234~thứ hai trăm ba mươi tư +thứ 1000~thứ một nghìn +thứ 1234~thứ một nghìn hai trăm ba mươi tư +hôm nay thứ 2~hôm nay thứ hai +đứng thứ 15~đứng thứ mười lăm \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_range.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_range.txt new file mode 100644 index 000000000..ea858dc4f --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_range.txt @@ -0,0 +1,11 @@ +10k-20k~mười nghìn đến hai mươi nghìn +1k-5k~một nghìn đến năm nghìn +2tr-5tr~hai triệu đến năm triệu +10$-20$~mười đô la đến hai mươi đô la +1t-2t~một tỷ đến hai tỷ +10:00-11:00~mười giờ đến mười một giờ +10$-20$~mười đô la đến hai mươi đô la +50.000đ-100.000đ~năm mươi nghìn đồng đến một trăm nghìn đồng +3kg-6kg~ba ki lô gam đến sáu ki lô gam +15cm-25cm~mười lăm xăng ti mét đến hai mươi lăm xăng ti mét +31Mhz-44Mhz~ba mươi mốt mê ga hẹc đến bốn mươi tư mê ga hẹc \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..96d57c551 --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_roman.txt @@ -0,0 +1,57 @@ +thế kỉ XV~thế kỉ mười lăm +thế kỉ XX~thế kỉ hai mươi +thế kỉ XXI~thế kỉ hai mươi mốt +thế kỷ IV~thế kỷ bốn +thế kỷ V~thế kỷ năm +thứ I~thứ một +thứ V~thứ năm +thứ X~thứ mười +thứ XV~thứ mười lăm +chương III~chương ba +phần ix~phần chín +chương C~chương một trăm +mục XCIX~mục chín mươi chín +chương MMMCMXCIX~chương ba nghìn chín trăm chín mươi chín +thế kỉ xix~thế kỉ mười chín +thế kỷ vi~thế kỷ sáu +phần xl~phần bốn mươi +mục xc~mục chín mươi +mục cd~mục bốn trăm +mục cm~mục chín trăm +thứ viii~thứ tám +thứ ix~thứ chín +thứ xi~thứ mười một +chương lxxxviii~chương tám mươi tám +chương cccxlv~chương ba trăm bốn mươi lăm +thế kỉ XV và chương IX~thế kỉ mười lăm và chương chín +trong phần X có mục IV~trong phần mười có mục bốn +chương I~chương một +chương MMMCMXCIX~chương ba nghìn chín trăm chín mươi chín +đoạn II~đoạn hai +đoạn iv~đoạn bốn +đoạn VII~đoạn bảy +đoạn xii~đoạn mười hai +năm MCMXCIX~năm một nghìn chín trăm chín mươi chín +năm mmxx~năm hai nghìn hai mươi +khoản III~khoản ba +khoản vi~khoản sáu +khoản XIV~khoản mười bốn +khoản xxv~khoản hai mươi lăm +phụ lục I~phụ lục một +phụ lục v~phụ lục năm +phụ lục XII~phụ lục mười hai +phụ lục xx~phụ lục hai mươi +khóa VII~khóa bảy +khóa xi~khóa mười một +khóa XV~khóa mười lăm +khóa xxx~khóa ba mươi +số I~số một +số v~số năm +số X~số mười +số l~số năm mươi +đoạn IX mục III~đoạn chín mục ba +khoản II phụ lục IV~khoản hai phụ lục bốn +khóa XII số IX~khóa mười hai số chín +năm MMXXIII khoản V~năm hai nghìn hai mươi ba khoản năm +chương VII đoạn XI~chương bảy đoạn mười một +phần XX mục XV~phần hai mươi mục mười lăm \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_time.txt new file mode 100644 index 000000000..4f576e821 --- /dev/null +++ b/tests/nemo_text_processing/vi/data_text_normalization/test_cases_time.txt @@ -0,0 +1,117 @@ +1h~một giờ +2h~hai giờ +3h~ba giờ +4h~bốn giờ +5h~năm giờ +6h~sáu giờ +7h~bảy giờ +8h~tám giờ +9h~chín giờ +10h~mười giờ +11h~mười một giờ +12h~mười hai giờ +13h~mười ba giờ +14h~mười bốn giờ +15h~mười lăm giờ +16h~mười sáu giờ +17h~mười bảy giờ +18h~mười tám giờ +19h~mười chín giờ +20h~hai mươi giờ +21h~hai mươi mốt giờ +22h~hai mươi hai giờ +23h~hai mươi ba giờ +24h~hai mươi tư giờ +0h~không giờ +01h~một giờ +02h~hai giờ +03h~ba giờ +04h~bốn giờ +05h~năm giờ +06h~sáu giờ +07h~bảy giờ +08h~tám giờ +09h~chín giờ +8:00~tám giờ +8:05~tám giờ năm phút +8:10~tám giờ mười phút +8:15~tám giờ mười lăm phút +8:20~tám giờ hai mươi phút +8:25~tám giờ hai mươi lăm phút +8:30~tám giờ ba mươi phút +8:35~tám giờ ba mươi lăm phút +8:40~tám giờ bốn mươi phút +8:45~tám giờ bốn mươi lăm phút +8:50~tám giờ năm mươi phút +8:55~tám giờ năm mươi lăm phút +8:01~tám giờ một phút +8:02~tám giờ hai phút +8:03~tám giờ ba phút +8:07~tám giờ bảy phút +8:09~tám giờ chín phút +01:00~một giờ +01:05~một giờ năm phút +01:30~một giờ ba mươi phút +02:15~hai giờ mười lăm phút +03:45~ba giờ bốn mươi lăm phút +14:30~mười bốn giờ ba mươi phút +15:45~mười lăm giờ bốn mươi lăm phút +23:59~hai mươi ba giờ năm mươi chín phút +00:00~không giờ +00:30~không giờ ba mươi phút +5:20:35~năm giờ hai mươi phút ba mươi lăm giây +6:10:05~sáu giờ mười phút năm giây +1:01:01~một giờ một phút một giây +8:00:00~tám giờ +12:30:45~mười hai giờ ba mươi phút bốn mươi lăm giây +23:59:59~hai mươi ba giờ năm mươi chín phút năm mươi chín giây +01:01:01~một giờ một phút một giây +02:02:02~hai giờ hai phút hai giây +03:00:03~ba giờ ba giây +04:30:00~bốn giờ ba mươi phút +1p~một phút +3p~ba phút +5p~năm phút +10p~mười phút +30p~ba mươi phút +59p~năm mươi chín phút +1s~một giây +30s~ba mươi giây +59s~năm mươi chín giây +3p20s~ba phút hai mươi giây +4p30s~bốn phút ba mươi giây +12p03s~mười hai phút ba giây +59p59s~năm mươi chín phút năm mươi chín giây +8p45s~tám phút bốn mươi lăm giây +01p01s~một phút một giây +02p30s~hai phút ba mươi giây +05p00s~năm phút +8:23 gmt~tám giờ hai mươi ba phút GMT +15h cst~mười lăm giờ CST +9:00 utc~chín giờ UTC +14:30 pst~mười bốn giờ ba mươi phút PST +20:15 est~hai mươi giờ mười lăm phút EST +12h jst~mười hai giờ JST +14g30~mười bốn giờ ba mươi phút +14h30~mười bốn giờ ba mươi phút +09g05~chín giờ năm phút +09h05~chín giờ năm phút +1 giờ~một giờ +2 giờ~hai giờ +10 giờ~mười giờ +14 giờ~mười bốn giờ +1 giờ 30 phút~một giờ ba mươi phút +14 giờ 30 phút~mười bốn giờ ba mươi phút +2 giờ 15 phút~hai giờ mười lăm phút +2 giờ 15 phút 10 giây~hai giờ mười lăm phút mười giây +14 giờ 30 phút 45 giây~mười bốn giờ ba mươi phút bốn mươi lăm giây +5 phút~năm phút +10 phút~mười phút +30 phút~ba mươi phút +5 phút 30 giây~năm phút ba mươi giây +10 phút 15 giây~mười phút mười lăm giây +10 giây~mười giây +30 giây~ba mươi giây +45 giây~bốn mươi lăm giây +14 giờ 30 phút UTC~mười bốn giờ ba mươi phút UTC +2 giờ 15 phút GMT~hai giờ mười lăm phút GMT \ No newline at end of file diff --git a/tests/nemo_text_processing/vi/test_cardinal.py b/tests/nemo_text_processing/vi/test_cardinal.py index 0a888f84b..00bafe3f1 100644 --- a/tests/nemo_text_processing/vi/test_cardinal.py +++ b/tests/nemo_text_processing/vi/test_cardinal.py @@ -15,29 +15,45 @@ import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer - - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestCardinal: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_cardinal.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_date.py b/tests/nemo_text_processing/vi/test_date.py index 90885b6e4..b3da475db 100644 --- a/tests/nemo_text_processing/vi/test_date.py +++ b/tests/nemo_text_processing/vi/test_date.py @@ -15,28 +15,45 @@ import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer - - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestDate: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_date.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_decimal.py b/tests/nemo_text_processing/vi/test_decimal.py index e1b246e1d..73ed99f54 100644 --- a/tests/nemo_text_processing/vi/test_decimal.py +++ b/tests/nemo_text_processing/vi/test_decimal.py @@ -15,28 +15,45 @@ import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer - - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestDecimal: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_decimal.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_decimal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_fraction.py b/tests/nemo_text_processing/vi/test_fraction.py index acd465cfd..efa35fcce 100644 --- a/tests/nemo_text_processing/vi/test_fraction.py +++ b/tests/nemo_text_processing/vi/test_fraction.py @@ -12,32 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. - import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file - -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestFraction: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_fraction.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_fraction.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_measure.py b/tests/nemo_text_processing/vi/test_measure.py index 991cbc487..4cb89cf80 100644 --- a/tests/nemo_text_processing/vi/test_measure.py +++ b/tests/nemo_text_processing/vi/test_measure.py @@ -20,6 +20,7 @@ try: from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer + from nemo_text_processing.text_normalization.normalize import Normalizer PYNINI_AVAILABLE = True except (ImportError, ModuleNotFoundError): @@ -41,3 +42,18 @@ class TestMeasure: def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.skipif( + not PYNINI_AVAILABLE, + reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", + ) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_money.py b/tests/nemo_text_processing/vi/test_money.py index c626eef41..bffbef614 100644 --- a/tests/nemo_text_processing/vi/test_money.py +++ b/tests/nemo_text_processing/vi/test_money.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,32 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. - +# pytest tests/nemo_text_processing/vi/test_money.py --cpu --cache-clear import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file - -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestMoney: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_money.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=True, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=True, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_ordinal.py b/tests/nemo_text_processing/vi/test_ordinal.py index 239234dda..9b15bd0c4 100644 --- a/tests/nemo_text_processing/vi/test_ordinal.py +++ b/tests/nemo_text_processing/vi/test_ordinal.py @@ -12,32 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. - import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file - -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestOrdinal: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_ordinal.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_ordinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_range.py b/tests/nemo_text_processing/vi/test_range.py new file mode 100644 index 000000000..7df7f9f9c --- /dev/null +++ b/tests/nemo_text_processing/vi/test_range.py @@ -0,0 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from tests.nemo_text_processing.utils import parse_test_case_file + + +class TestRange: + normalizer = Normalizer(input_case='cased', lang='vi', cache_dir=None, overwrite_cache=True) + + @parameterized.expand(parse_test_case_file("vi/data_text_normalization/test_cases_range.txt")) + @pytest.mark.run_only_on('CPU') + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input) + assert pred == expected, f"input: {test_input} assert {pred} == {expected}" diff --git a/tests/nemo_text_processing/vi/test_roman.py b/tests/nemo_text_processing/vi/test_roman.py new file mode 100644 index 000000000..a942eb140 --- /dev/null +++ b/tests/nemo_text_processing/vi/test_roman.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio + +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file + + +class TestRoman: + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_roman.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh index cc1defd46..751351cd4 100644 --- a/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/vi/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do diff --git a/tests/nemo_text_processing/vi/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/vi/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..c11d66ef0 --- /dev/null +++ b/tests/nemo_text_processing/vi/test_sparrowhawk_normalization.sh @@ -0,0 +1,87 @@ + +#! /bin/sh + +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests"} + +runtest () { + input=$1 + echo "INPUT is $input" + cd ${GRAMMARS_DIR} + + # read test file + while read testcase; do + IFS='~' read written spoken <<< $testcase + norm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + norm_pred="$(echo -e "${norm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$written" "$spoken" "$norm_pred" + done < "$input" +} + +testTNCardinal() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +testTNDate() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_date.txt + runtest $input +} + +testTNDecimal() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_decimal.txt + runtest $input +} + +testTNOrdinal() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_ordinal.txt + runtest $input +} + +testTNFraction() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_fraction.txt + runtest $input +} + +testTNRoman() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_roman.txt + runtest $input +} + +testTNTime() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_time.txt + runtest $input +} + +testTNMeasure() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_measure.txt + runtest $input +} + +testTNMoney() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_money.txt + runtest $input +} + +testTNRange() { + input=$PROJECT_DIR/vi/data_text_normalization/test_cases_range.txt + runtest $input +} + +# testTNTelephone() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_telephone.txt +# runtest $input +# } + +# testTNElectronic() { +# input=$PROJECT_DIR/vi/data_text_normalization/test_cases_electronic.txt +# runtest $input +# } + +# Load shUnit2 +. /workspace/shunit2/shunit2 diff --git a/tests/nemo_text_processing/vi/test_time.py b/tests/nemo_text_processing/vi/test_time.py index 9502cad54..44dfdd875 100644 --- a/tests/nemo_text_processing/vi/test_time.py +++ b/tests/nemo_text_processing/vi/test_time.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,28 +15,45 @@ import pytest from parameterized import parameterized -from ..utils import CACHE_DIR, parse_test_case_file +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer +from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio -try: - from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer - - PYNINI_AVAILABLE = True -except (ImportError, ModuleNotFoundError): - PYNINI_AVAILABLE = False +from ..utils import CACHE_DIR, RUN_AUDIO_BASED_TESTS, parse_test_case_file class TestTime: - inverse_normalizer = ( - InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) if PYNINI_AVAILABLE else None - ) + + inverse_normalizer = InverseNormalizer(lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('vi/data_inverse_text_normalization/test_cases_time.txt')) - @pytest.mark.skipif( - not PYNINI_AVAILABLE, - reason="`pynini` not installed, please install via nemo_text_processing/pynini_install.sh", - ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected + + normalizer = Normalizer( + input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=True + ) + + normalizer_with_audio = ( + NormalizerWithAudio(input_case='cased', lang='vi', cache_dir=CACHE_DIR, overwrite_cache=False) + if CACHE_DIR and RUN_AUDIO_BASED_TESTS + else None + ) + + @parameterized.expand(parse_test_case_file('vi/data_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" + + if self.normalizer_with_audio: + pred_non_deterministic = self.normalizer_with_audio.normalize( + test_input, + n_tagged=30, + punct_post_process=False, + ) + assert expected in pred_non_deterministic, f"input: {test_input}" diff --git a/tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..f36dc4293 --- /dev/null +++ b/tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,21 @@ +人力资源~HR +自动取款机~ATM +人力资源~HR +首席执行官~CEO +美国研究生入学考试~GRE +研究生管理专业入学考试~GMAT +全球定位系统~GPS +刷卡机~POS机 +数位多功能光碟~DVD +镭射唱片~CD +通用串行总线~USB +统一资源定位符~URL +虚拟专用网络~VPN +网络互联协议~IP +脱氧核糖核酸~DNA +核糖核酸~RNA +平均学分绩点~GPA +发光二极管~LED +可移植文档格式~PDF +社会性网络服务~SNS +博士~PhD diff --git a/tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_word.txt new file mode 100644 index 000000000..1d0cac255 --- /dev/null +++ b/tests/nemo_text_processing/zh/data_inverse_text_normalization/test_cases_word.txt @@ -0,0 +1,21 @@ +你好~你好 +年级~年级 +秘密~秘密 +键盘~键盘 +借口~借口 +学生~学生 +人力~人力 +转移~转移 +徘徊~徘徊 +冤枉~冤枉 +浏览~浏览 +珍藏~珍藏 +患难 ~患难 +湿~湿 +眼眶~眼眶 +遗产~遗产 +流浪~流浪 +信仰~信仰 +戒指~戒指 +义无反顾~义无反顾 +交换~交换 diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_cardinal.txt index 11f3f8cc1..573042f7a 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_cardinal.txt @@ -1,161 +1,85 @@ 10000~一万 -负10000~负一万 -正100000~正十万 +负1000~负一千 +100000~十万 210~二百一十 -负200~负二百 -负219~负二百一十九 +200~二百 +219~二百一十九 5000~五千 5100~五千一百 5110~五千一百一十 5111~五千一百一十一 5010~五千零一十 -5099~五千零九十九 +5010~五千零一十 5001~五千零一 -5,000~五千 -5,100~五千一百 -5,010~五千零一十 -5,001~五千零一 -50,000~五万 -51,000~五万一千 -50,100~五万零一百 -50,010~五万零一十 -51,100~五万一千一百 -51,110~五万一千一百一十 -50,011~五万零一十一 -50,001~五万零一 -50,010~五万零一十 50000~五万 51000~五万一千 50100~五万零一百 50010~五万零一十 -50001~五万零一 51100~五万一千一百 51110~五万一千一百一十 -51111~五万一千一百一十一 -50111~五万零一百一十一 50011~五万零一十一 -500,000~五十万 -510,000~五十一万 -501,000~五十万一千 -500,100~五十万零一百 -500,010~五十万零一十 -500,111~五十万零一百一十一 -501,100~五十万一千一百 -501,111~五十万一千一百一十一 -511,111~五十一万一千一百一十一 -100,000~十万 -110,000~十一万 +50001~五万零一 +50010~五万零一十 +50000~五万 +51000~五万一千 +50100~五万零一百 +50010~五万零一十 +50001~五万零一 +51100~五万一千一百 +50011~五万零一十一 +500000~五十万 +510000~五十一万 +501000~五十万一千 +500100~五十万零一百 +100000~十万 +110000~十一万 500000~五十万 510000~五十一万 501000~五十万一千 500100~五十万零一百 500010~五十万零一十 -500001~五十万零一 -500111~五十万零一百一十一 -501100~五十万一千一百 501111~五十万一千一百一十一 511111~五十一万一千一百一十一 100000~十万 110000~十一万 1100000~一百一十万 -1010000~一百零一万 -1001000~一百万一千 -1000100~一百万零一百 -1000010~一百万零一十 1000001~一百万零一 1000000~一百万 -1,000,000~一百万 -1,100,000~一百一十万 -1,010,000~一百零一万 -1,001,000~一百万一千 -1,000,100~一百万零一百 -1,000,010~一百万零一十 -1,000,001~一百万零一 10000000~一千万 11000000~一千一百万 10100000~一千零一十万 10010000~一千零一万 +11000000~一千一百万 +10100000~一千零一十万 +10010000~一千零一万 10001000~一千万一千 -10010100~一千零一万零一百 -10010010~一千零一万零一十 -10010001~一千零一万零一 -10,000,000~一千万 -11,000,000~一千一百万 -10,100,000~一千零一十万 -10,010,000~一千零一万 -10,001,000~一千万一千 -10,010,100~一千零一万零一百 -10,010,010~一千零一万零一十 -10,010,001~一千零一万零一 -101111111~一亿一百一十一万一千一百一十一 +101111111~一亿零一百一十一万一千一百一十一 110111111~一亿一千零一十一万一千一百一十一 111011111~一亿一千一百零一万一千一百一十一 111101111~一亿一千一百一十万一千一百一十一 111110111~一亿一千一百一十一万零一百一十一 -111111011~一亿一千一百一十一万一千零一十一 -111111101~一亿一千一百一十一万一千一百零一 -111111111~一亿一千一百一十一万一千一百一十一 -101,111,111~一亿一百一十一万一千一百一十一 -110,111,111~一亿一千零一十一万一千一百一十一 -111,011,111~一亿一千一百零一万一千一百一十一 -111,101,111~一亿一千一百一十万一千一百一十一 -111,110,111~一亿一千一百一十一万零一百一十一 -111,111,011~一亿一千一百一十一万一千零一十一 -111,111,101~一亿一千一百一十一万一千一百零一 -111,111,110~一亿一千一百一十一万一千一百一十 -111,111,111~一亿一千一百一十一万一千一百一十一 +101111111~一亿零一百一十一万一千一百一十一 +110111111~一亿一千零一十一万一千一百一十一 +111011111~一亿一千一百零一万一千一百一十一 +111101111~一亿一千一百一十万一千一百一十一 1011111111~十亿一千一百一十一万一千一百一十一 -1101111111~十一亿一百一十一万一千一百一十一 -1110111111~十一亿一千零一十一万一千一百一十一 +1101111111~十一亿零一百一十一万一千一百一十一 1111011111~十一亿一千一百零一万一千一百一十一 -1111110111~十一亿一千一百一十一万零一百一十一 -1111111101~十一亿一千一百一十一万一千一百零一 -1111111111~十一亿一千一百一十一万一千一百一十一 -1,011,111,111~十亿一千一百一十一万一千一百一十一 -1,101,111,111~十一亿一百一十一万一千一百一十一 -1,110,111,111~十一亿一千零一十一万一千一百一十一 -1,111,011,111~十一亿一千一百零一万一千一百一十一 -1,111,101,111~十一亿一千一百一十万一千一百一十一 -1,111,110,111~十一亿一千一百一十一万零一百一十一 -1,111,111,011~十一亿一千一百一十一万一千零一十一 -1,111,111,101~十一亿一千一百一十一万一千一百零一 -1,111,111,110~十一亿一千一百一十一万一千一百一十 +11000000000~一百一十亿 +10000100000~一百亿零十万 11000000000~一百一十亿 10100000000~一百零一亿 10010000000~一百亿一千万 -10001000000~一百亿一百万 -10000100000~一百亿零十万 -11,000,000,000~一百一十亿 -10,100,000,000~一百零一亿 -10,010,000,000~一百亿一千万 -10,001,000,000~一百亿一百万 -10,000,100,000~一百亿零十万 -10,000,010,000~一百亿零一万 -10,000,001,000~一百亿零一千 -10,000,000,100~一百亿零一百 -10,000,000,010~一百亿零一十 -10,000,000,001~一百亿零一 +10001000000~一百亿零一百万 +10000000001~一百亿零一 100000100000~一千亿零十万 100000010000~一千亿零一万 100000001000~一千亿零一千 -100000000100~一千亿零一百 -100000000010~一千亿零一十 -100000000001~一千亿零一 -100,000,000,000~一千亿 -110,000,000,000~一千一百亿 -101,000,000,000~一千零一十亿 -100,100,000,000~一千零一亿 -100,010,000,000~一千亿一千万 -100,001,000,000~一千亿一百万 -100,000,100,000~一千亿零十万 -100,000,010,000~一千亿零一万 -100,000,001,000~一千亿零一千 -100,000,000,100~一千亿零一百 -20万~二十万 -5万~五万 -100万~一百万 -1500万~一千五百万 -20亿~二十亿 -100亿~一百亿 -1500亿~一千五百亿 -9亿~九亿 +101000000000~一千零一十亿 +100100000000~一千零一亿 +我今天买了5个苹果~我今天买了五个苹果 +我今天买了25个苹果~我今天买了二十五个苹果 +我今天买了35个苹果~我今天买了三十五个苹果 +我今天买了50000个苹果~我今天买了五万个苹果 +我今天买了150000个苹果~我今天买了十五万个苹果 +双辽境内除东辽河、西辽河等5条河流~双辽境内除东辽河、西辽河等五条河流 diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_date.txt index c0963e9d0..52ab15f44 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_date.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_date.txt @@ -27,4 +27,9 @@ 2020年11月24日~二零二零年十一月二十四日 公元2020年11月24日~公元二零二零年十一月二十四日 1823年3月bc~公元前一八二三年三月 -纪元2013年~纪元二零一三年 \ No newline at end of file +纪元2013年~纪元二零一三年 +今天是2013年3月3日~今天是二零一三年三月三日 +现在是12月25日圣诞节~现在是十二月二十五日圣诞节 +文件上的标注日期是12/2/2~文件上的标注日期是一二年二月二日 +现在是入冬的12月~现在是入冬的十二月 +公元前202年西汉成立~公元前二零二年西汉成立 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_decimal.txt index 10e49a97f..4d53200fd 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_decimal.txt @@ -5,26 +5,31 @@ -5.5555~负五点五五五五 1890.5555~一千八百九十点五五五五 20.123~二十点一二三 --2930.1929~负二千九百三十点一九二九 +-2930.1929~负两千九百三十点一九二九 0.5~零点五 5.0~五点零 10.567~十点五六七 -123.123~负一百二十三点一二三 3123.1231~三千一百二十三点一二三一 -123123123.12312334234~一亿二千三百一十二万三千一百二十三点一二三一二三三四二三四 +123123123.12312334234~一亿两千三百一十二万三千一百二十三点一二三一二三三四二三四 83888123.2398412~八千三百八十八万八千一百二十三点二三九八四一二 283818.28994万~二十八万三千八百一十八点二八九九四万 -28394919.2312亿~二千八百三十九万四千九百一十九点二三一二亿 +28394919.2312亿~两千八百三十九万四千九百一十九点二三一二亿 23.23万~二十三点二三万 1233.1亿~一千二百三十三点一亿 -123.213万~负一百二十三点二一三万 123.890万~一百二十三点八九零万 -233123.9940亿~负二十三万三千一百二十三点九九四零亿 283943.234123亿~二十八万三千九百四十三点二三四一二三亿 -2391.2318~二千三百九十一点二三一八 +2391.2318~两千三百九十一点二三一八 -1.5亿~负一点五亿 1.5亿~一点五亿 10.67亿~十点六七亿 16.3亿~十六点三亿 12.2亿~十二点二亿 -2342.2342亿~二千三百四十二点二三四二亿 \ No newline at end of file +2342.2342亿~两千三百四十二点二三四二亿 +公司的年收益率是6.5~公司的年收益率是六点五 +现在的室外气温是36.7摄氏度~现在的室外气温是三十六点七摄氏度 +我们可以给你返还1.2个百分点~我们可以给你返还一点二个百分点 +全球现今有71.5亿人~全球现今有七十一点五亿人 +小张的资产值13.5亿~小张的资产值十三点五亿 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_fraction.txt index bae47330d..c39f680f4 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_fraction.txt @@ -4,11 +4,7 @@ -1/3~负三分之一 负1/3~负三分之一 1/2~二分之一 -+1/2~正二分之一 -正1/2~正二分之一 1/10~十分之一 -+100分之1~正一百分之一 -正100分之1~正一百分之一 98%~百分之九十八 -98%~负百分之九十八 负98%~负百分之九十八 @@ -16,19 +12,15 @@ -百分之1~负百分之一 负百分之1~负百分之一 -100分之57~负一百分之五十七 -正100分之57~正一百分之五十七 负100分之57~负一百分之五十七 1/5~五分之一 -1/5~负五分之一 -+1/5~正五分之一 1又1/5~一又五分之一 -+1又1/5~正一又五分之一 -正1又1/5~正一又五分之一 5又2分之1~五又二分之一 -+5又2分之1~正五又二分之一 -正5又2分之1~正五又二分之一 -1/100~一百分之一 -+1/100~正一百分之一 -正1/100~正一百分之一 +0.4%~百分之零点四 6.3%~百分之六点三 -0.4%~百分之零点四 \ No newline at end of file +公司的年收益率是6.5%~公司的年收益率是百分之六点五 +60%的人口是男性~百分之六十的人口是男性 +全校有80%的学生来自大山深处的贫困地区~全校有百分之八十的学生来自大山深处的贫困地区 +我们的队伍有1/3的人是士官学校出身~我们的队伍有三分之一的人是士官学校出身 +今年的降雨量较往年多了5%~今年的降雨量较往年多了百分之五 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_math.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_math.txt deleted file mode 100644 index d31a3a8d4..000000000 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_math.txt +++ /dev/null @@ -1,2 +0,0 @@ -78:96~七十八比九十六 -±2~正负二 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_measure.txt index 094afc7c4..d220c406a 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_measure.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_measure.txt @@ -2,4 +2,10 @@ 38°C~三十八摄氏度 120m²~一百二十平方米 10ms~十毫秒 +25千克~二十五千克 -23°C~负二十三摄氏度 +1.2g~一点二克 +测量机显示重量是25kg~测量机显示重量是二十五千克 +现在的室外温度是38°C~现在的室外温度是三十八摄氏度 +这个房子大概有120m²~这个房子大概有一百二十平方米 +整体时长大概是10ms~整体时长大概是十毫秒 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_money.txt index 28075fca1..71dd98d71 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_money.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_money.txt @@ -12,7 +12,11 @@ $100~一百美元 5角~五角 5块~五块 6毛~六毛 -5块5毛5分~五块五毛五分 -1.5万美元~一点五万美元 $1.5万~一点五万美元 -3.5万韩元~三点五万韩元 \ No newline at end of file +3.5万韩元~三点五万韩元 +1.5万美元~一点五万美元 +我现在能拿出的现金是100000美元~我现在能拿出的现金是十万美元 +一份煎饼卖5块一份~一份煎饼卖五块一份 +每100美元能兑换700人民币左右~每一百美元能兑换七百人民币左右 +您的银行账户余额为$500~您的银行账户余额为五百美元 +洛杉矶的最低工资是每小时$15~洛杉矶的最低工资是每小时十五美元 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_ordinal.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_ordinal.txt index 57ea76bc1..e84b3dd8d 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_ordinal.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_ordinal.txt @@ -1,149 +1,84 @@ 第10000~第一万 第210~第二百一十 第5000~第五千 -第5100~第五千一百 -第5110~第五千一百一十 -第5111~第五千一百一十一 -第5010~第五千零一十 第5099~第五千零九十九 第5001~第五千零一 -第5,000~第五千 -第5,100~第五千一百 -第5,010~第五千零一十 -第5,001~第五千零一 -第50,000~第五万 -第51,000~第五万一千 -第50,100~第五万零一百 -第50,010~第五万零一十 -第51,100~第五万一千一百 -第51,110~第五万一千一百一十 -第50,011~第五万零一十一 -第50,001~第五万零一 -第50,010~第五万零一十 +第5000~第五千 +第5100~第五千一百 +第50010~第五万零一十 +第51100~第五万一千一百 +第50010~第五万零一十 第50000~第五万 第51000~第五万一千 第50100~第五万零一百 第50010~第五万零一十 -第50001~第五万零一 -第51100~第五万一千一百 -第51110~第五万一千一百一十 -第51111~第五万一千一百一十一 第50111~第五万零一百一十一 第50011~第五万零一十一 -第500,000~第五十万 -第510,000~第五十一万 -第501,000~第五十万一千 -第500,100~第五十万零一百 -第500,010~第五十万零一十 -第500,111~第五十万零一百一十一 -第501,100~第五十万一千一百 -第501,111~第五十万一千一百一十一 -第511,111~第五十一万一千一百一十一 -第100,000~第十万 -第110,000~第十一万 +第500000~第五十万 +第510000~第五十一万 +第501000~第五十万一千 +第100000~第十万 +第110000~第十一万 第500000~第五十万 第510000~第五十一万 第501000~第五十万一千 第500100~第五十万零一百 -第500010~第五十万零一十 -第500001~第五十万零一 -第500111~第五十万零一百一十一 -第501100~第五十万一千一百 第501111~第五十万一千一百一十一 第511111~第五十一万一千一百一十一 第100000~第十万 -第110000~第十一万 -第1100000~第一百一十万 -第1010000~第一百零一万 -第1001000~第一百万一千 第1000100~第一百万零一百 第1000010~第一百万零一十 第1000001~第一百万零一 第1000000~第一百万 -第1,000,000~第一百万 -第1,100,000~第一百一十万 -第1,010,000~第一百零一万 -第1,001,000~第一百万一千 -第1,000,100~第一百万零一百 -第1,000,010~第一百万零一十 -第1,000,001~第一百万零一 +第1000100~第一百万零一百 +第1000010~第一百万零一十 +第1000001~第一百万零一 +第10000000~第一千万 +第11000000~第一千一百万 +第10010001~第一千零一万零一 第10000000~第一千万 第11000000~第一千一百万 -第10100000~第一千零一十万 -第10010000~第一千零一万 -第10001000~第一千万一千 -第10010100~第一千零一万零一百 -第10010010~第一千零一万零一十 第10010001~第一千零一万零一 -第10,000,000~第一千万 -第11,000,000~第一千一百万 -第10,100,000~第一千零一十万 -第10,010,000~第一千零一万 -第10,001,000~第一千万一千 -第10,010,100~第一千零一万零一百 -第10,010,010~第一千零一万零一十 -第10,010,001~第一千零一万零一 -第101111111~第一亿一百一十一万一千一百一十一 +第101111111~第一亿零一百一十一万一千一百一十一 第110111111~第一亿一千零一十一万一千一百一十一 第111011111~第一亿一千一百零一万一千一百一十一 第111101111~第一亿一千一百一十万一千一百一十一 -第111110111~第一亿一千一百一十一万零一百一十一 -第111111011~第一亿一千一百一十一万一千零一十一 -第111111101~第一亿一千一百一十一万一千一百零一 +第101111111~第一亿零一百一十一万一千一百一十一 +第110111111~第一亿一千零一十一万一千一百一十一 +第111111110~第一亿一千一百一十一万一千一百一十 第111111111~第一亿一千一百一十一万一千一百一十一 -第101,111,111~第一亿一百一十一万一千一百一十一 -第110,111,111~第一亿一千零一十一万一千一百一十一 -第111,011,111~第一亿一千一百零一万一千一百一十一 -第111,101,111~第一亿一千一百一十万一千一百一十一 -第111,110,111~第一亿一千一百一十一万零一百一十一 -第111,111,011~第一亿一千一百一十一万一千零一十一 -第111,111,101~第一亿一千一百一十一万一千一百零一 -第111,111,110~第一亿一千一百一十一万一千一百一十 -第111,111,111~第一亿一千一百一十一万一千一百一十一 第1011111111~第十亿一千一百一十一万一千一百一十一 -第1101111111~第十一亿一百一十一万一千一百一十一 +第1101111111~第十一亿零一百一十一万一千一百一十一 第1110111111~第十一亿一千零一十一万一千一百一十一 第1111011111~第十一亿一千一百零一万一千一百一十一 第1111110111~第十一亿一千一百一十一万零一百一十一 第1111111101~第十一亿一千一百一十一万一千一百零一 第1111111111~第十一亿一千一百一十一万一千一百一十一 -第1,011,111,111~第十亿一千一百一十一万一千一百一十一 -第1,101,111,111~第十一亿一百一十一万一千一百一十一 -第1,110,111,111~第十一亿一千零一十一万一千一百一十一 -第1,111,011,111~第十一亿一千一百零一万一千一百一十一 -第1,111,101,111~第十一亿一千一百一十万一千一百一十一 -第1,111,110,111~第十一亿一千一百一十一万零一百一十一 -第1,111,111,011~第十一亿一千一百一十一万一千零一十一 -第1,111,111,101~第十一亿一千一百一十一万一千一百零一 -第1,111,111,110~第十一亿一千一百一十一万一千一百一十 +第1011111111~第十亿一千一百一十一万一千一百一十一 +第1101111111~第十一亿零一百一十一万一千一百一十一 +第1110111111~第十一亿一千零一十一万一千一百一十一 +第11000000000~第一百一十亿 +第10000100000~第一百亿零十万 第11000000000~第一百一十亿 第10100000000~第一百零一亿 第10010000000~第一百亿一千万 -第10001000000~第一百亿一百万 +第10001000000~第一百亿零一百万 第10000100000~第一百亿零十万 -第11,000,000,000~第一百一十亿 -第10,100,000,000~第一百零一亿 -第10,010,000,000~第一百亿一千万 -第10,001,000,000~第一百亿一百万 -第10,000,100,000~第一百亿零十万 -第10,000,010,000~第一百亿零一万 -第10,000,001,000~第一百亿零一千 -第10,000,000,100~第一百亿零一百 -第10,000,000,010~第一百亿零一十 -第10,000,000,001~第一百亿零一 +第10000000100~第一百亿零一百 +第10000000010~第一百亿零一十 +第10000000001~第一百亿零一 第100000100000~第一千亿零十万 第100000010000~第一千亿零一万 第100000001000~第一千亿零一千 -第100000000100~第一千亿零一百 第100000000010~第一千亿零一十 第100000000001~第一千亿零一 -第100,000,000,000~第一千亿 -第110,000,000,000~第一千一百亿 -第101,000,000,000~第一千零一十亿 -第100,100,000,000~第一千零一亿 -第100,010,000,000~第一千亿一千万 -第100,001,000,000~第一千亿一百万 -第100,000,100,000~第一千亿零十万 -第100,000,010,000~第一千亿零一万 -第100,000,001,000~第一千亿零一千 -第100,000,000,100~第一千亿零一百 \ No newline at end of file +第100000000000~第一千亿 +第110000000000~第一千一百亿 +第101000000000~第一千零一十亿 +第100100000000~第一千零一亿 +第100010000000~第一千亿一千万 +这个孩子的学习成绩一直是全年级第1~这个孩子的学习成绩一直是全年级第一 +从这一排往下数第5个就是小明~从这一排往下数第五个就是小明 +恭喜您成为本店第100名顾客~恭喜您成为本店第一百名顾客 +这是你人生的第1桶金~这是你人生的第一桶金 +这个名单从头开始到第100都是你的目标客户~这个名单从头开始到第一百都是你的目标客户 diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_preprocess.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_preprocess.txt deleted file mode 100644 index e1b592ebc..000000000 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_preprocess.txt +++ /dev/null @@ -1 +0,0 @@ -你啊好~你好 diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_time.txt index 94b45ac30..9523492a0 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_time.txt @@ -1,4 +1,4 @@ -3:4:5~三点四分五秒 +03:04:05~三点四分五秒 03:04:05~三点四分五秒 00:00:00~零点零分零秒 03:04:05~三点四分五秒 @@ -27,4 +27,8 @@ 5点差3分~五点差三分 5点差5分~五点差五分 5点差4分am~五点差四分am -3个小时15分钟30秒~三个小时十五分钟三十秒 \ No newline at end of file +3个小时15分钟30秒~三个小时十五分钟三十秒 +现在是北京时间下午03:04:05~现在是北京时间下午三点四分五秒 +航班预计会延误5个小时~航班预计会延误五个小时 +大家尽量把手表对准调到5点1刻~大家尽量把手表对准调到五点一刻 +5点1刻离六点就差十五分钟~五点一刻离六点就差十五分钟 \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..1700f1af6 --- /dev/null +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,10 @@ +这附近有Atm~这附近有ATM +这是一个ufo的照片~这是一个UFO的照片 +nba比赛如期举行~NBA比赛如期举行 +我们需要升级gpu~我们需要升级GPU +他是这个公司的c e o~他是这个公司的CEO +我们已经加入了wto~我们已经加入了WTO +小王以优秀的战绩成为这场游戏的mvp~小王以优秀的战绩成为这场游戏的MVP +这位客人是我们的vip~这位客人是我们的VIP +小王的iq是全班最高的~小王的IQ是全班最高的 +小李读了一个mba~小李读了一个MBA \ No newline at end of file diff --git a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_word.txt b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_word.txt index 4fedd2cd9..81ef6cdb3 100644 --- a/tests/nemo_text_processing/zh/data_text_normalization/test_cases_word.txt +++ b/tests/nemo_text_processing/zh/data_text_normalization/test_cases_word.txt @@ -1,2 +1,25 @@ 你~你 -好~好 \ No newline at end of file +好~好 +你好今天的天气不错~你好今天的天气不错 +只有智商超过一定数值的人才能破解~只有智商超过一定数值的人才能破解 +这是由人工智能控制的系统~这是由人工智能控制的系统 +欧洲旅游目的地多到不知道怎么选~欧洲旅游目的地多到不知道怎么选 +马斯科卖掉豪宅住进折叠屋~马斯科卖掉豪宅住进折叠屋 +免除GOOGLE在一桩诽谤官司中的法律责任。~免除GOOGLE在一桩诽谤官司中的法律责任。 +这对CHROME是有利的。~这对CHROME是有利的。 +这可能是PILde使用者。~这可能是PILde使用者。 +CSI侧重科学办案，也就是现场搜正和鉴识。~CSI侧重科学办案，也就是现场搜正和鉴识。 +我以前非常喜欢一个软体，DRAW。~我以前非常喜欢一个软体，DRAW。 +我爱你病毒。~我爱你病毒。 +微软举办了RACETOMARKETCHALLENGE竞赛。~微软举办了RACETOMARKETCHALLENGE竞赛。 +苹果销售量的复苏程度远超PC市场。~苹果销售量的复苏程度远超PC市场。 +第三季还有两款ANDROID手机亮相。~第三季还有两款ANDROID手机亮相。 +反而应试著让所有GOOGLE服务更加社交化。~反而应试著让所有GOOGLE服务更加社交化。 +GOOGLE已提供一项NATIVECLIENT软体。~GOOGLE已提供一项NATIVECLIENT软体。 +这些程式都支援PRE与ITUNES同步化。~这些程式都支援PRE与ITUNES同步化。 +可以推断此次NTT可能也会将同样的策略用在LTE上。~可以推断此次NTT可能也会将同样的策略用在LTE上。 +现今许多小型企业因成本考量被迫采用一般PC作为伺服器。~现今许多小型企业因成本考量被迫采用一般PC作为伺服器。 +部落格宣布GOOGLECHROMES的诞生。~部落格宣布GOOGLECHROMES的诞生。 +由ZIP订购机场接送或观光景点共乘服务。~由ZIP订购机场接送或观光景点共乘服务。 +PAQUE表示短时间应该还不会全面开放。~PAQUE表示短时间应该还不会全面开放。 +CBS是美国一家重要的广播电视网路公司。~CBS是美国一家重要的广播电视网路公司。 diff --git a/tests/nemo_text_processing/zh/test_cardinal.py b/tests/nemo_text_processing/zh/test_cardinal.py index d09b71f3d..a8274d398 100644 --- a/tests/nemo_text_processing/zh/test_cardinal.py +++ b/tests/nemo_text_processing/zh/test_cardinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/zh/test_date.py b/tests/nemo_text_processing/zh/test_date.py index 01d3e038b..1621ce5e5 100644 --- a/tests/nemo_text_processing/zh/test_date.py +++ b/tests/nemo_text_processing/zh/test_date.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/zh/test_decimal.py b/tests/nemo_text_processing/zh/test_decimal.py index db6046b60..1846efa54 100644 --- a/tests/nemo_text_processing/zh/test_decimal.py +++ b/tests/nemo_text_processing/zh/test_decimal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/zh/test_fraction.py b/tests/nemo_text_processing/zh/test_fraction.py index 264d64d13..9b71f1d06 100644 --- a/tests/nemo_text_processing/zh/test_fraction.py +++ b/tests/nemo_text_processing/zh/test_fraction.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/zh/test_measure.py b/tests/nemo_text_processing/zh/test_measure.py index 32df28855..fd8d76aa3 100644 --- a/tests/nemo_text_processing/zh/test_measure.py +++ b/tests/nemo_text_processing/zh/test_measure.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/zh/test_money.py b/tests/nemo_text_processing/zh/test_money.py index 3d50ce5fa..8aa2fc320 100644 --- a/tests/nemo_text_processing/zh/test_money.py +++ b/tests/nemo_text_processing/zh/test_money.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/zh/test_ordinal.py b/tests/nemo_text_processing/zh/test_ordinal.py index f8644c31d..409e62964 100644 --- a/tests/nemo_text_processing/zh/test_ordinal.py +++ b/tests/nemo_text_processing/zh/test_ordinal.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file diff --git a/tests/nemo_text_processing/zh/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/zh/test_sparrowhawk_inverse_text_normalization.sh index 4ca12af7f..708f54dd5 100644 --- a/tests/nemo_text_processing/zh/test_sparrowhawk_inverse_text_normalization.sh +++ b/tests/nemo_text_processing/zh/test_sparrowhawk_inverse_text_normalization.sh @@ -1,10 +1,12 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests/en"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do @@ -21,62 +23,62 @@ runtest () { } testITNCardinal() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_cardinal.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_cardinal.txt runtest $input } testITNDate() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_date.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_date.txt runtest $input } testITNDecimal() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_decimal.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_decimal.txt runtest $input } testITNOrdinal() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_ordinal.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_ordinal.txt runtest $input } testITNFraction() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_fraction.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_fraction.txt runtest $input } testITNTime() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_time.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_time.txt runtest $input } -testITNMeasure() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_measure.txt - runtest $input -} +#testITNMeasure() { +# input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_measure.txt +# runtest $input +#} testITNMoney() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_money.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_money.txt runtest $input } testITNWhitelist() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_whitelist.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_whitelist.txt runtest $input } -testITNTelephone() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_telephone.txt - runtest $input -} +#testITNTelephone() { +# input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_telephone.txt +# runtest $input +#} -testITNElectronic() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_electronic.txt - runtest $input -} +#testITNElectronic() { +# input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_electronic.txt +# runtest $input +#} testITNWord() { - input=$PROJECT_DIR/fr/data_inverse_text_normalization/test_cases_word.txt + input=$PROJECT_DIR/zh/data_inverse_text_normalization/test_cases_word.txt runtest $input } diff --git a/tests/nemo_text_processing/zh/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/zh/test_sparrowhawk_normalization.sh index 6c3a6598f..5089427b6 100644 --- a/tests/nemo_text_processing/zh/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/zh/test_sparrowhawk_normalization.sh @@ -1,16 +1,18 @@ #! /bin/sh -PROJECT_DIR=/workspace/tests +GRAMMARS_DIR=${1:-"/workspace/sparrowhawk/documentation/grammars"} +PROJECT_DIR=${2:-"/workspace/tests"} runtest () { input=$1 - cd /workspace/sparrowhawk/documentation/grammars + echo "INPUT is $input" + cd ${GRAMMARS_DIR} # read test file while read testcase; do IFS='~' read written spoken <<< $testcase # replace non breaking space with breaking space - denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration_pp.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') # # trim white space spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" @@ -21,35 +23,42 @@ runtest () { done < "$input" } -testTNMoneyText() { - input=$PROJECT_DIR/zh/data_text_normalization/test_cases_money.txt +testTNTimeText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_time.txt runtest $input } -testTNCharText() { - input=$PROJECT_DIR/zh/data_text_normalization/test_cases_char.txt +testTNCardinalText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_cardinal.txt runtest $input } -testTNTimeText() { - input=$PROJECT_DIR/zh/data_text_normalization/test_cases_time.txt +testTNOrdinalText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_ordinal.txt runtest $input } -testTNDateText() { - input=$PROJECT_DIR/zh/data_text_normalization/test_cases_date.txt +testTNDecimalalText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_decimal.txt runtest $input } -# testTNMathText() { -# input=$PROJECT_DIR/zh/data_text_normalization/test_cases_math.txt -# runtest $input -# } testTNFractionText() { input=$PROJECT_DIR/zh/data_text_normalization/test_cases_fraction.txt runtest $input } - -# testTNPreprocessText() { -# input=$PROJECT_DIR/zh/data_text_normalization/test_cases_preprocess.txt -# runtest $input -# } +testTNDateText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_date.txt + runtest $input +} +testTNMoneyText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_money.txt + runtest $input +} +testTNWordText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_word.txt + runtest $input +} +testTNWhitelistText() { + input=$PROJECT_DIR/zh/data_text_normalization/test_cases_whitelist.txt + runtest $input +} testTNMeasureText() { input=$PROJECT_DIR/zh/data_text_normalization/test_cases_measure.txt runtest $input @@ -57,4 +66,5 @@ testTNMeasureText() { # Load shUnit2 -. $PROJECT_DIR/../shunit2/shunit2 +#. $PROJECT_DIR/../shunit2/shunit2 +. /workspace/shunit2/shunit2 diff --git a/tests/nemo_text_processing/zh/test_time.py b/tests/nemo_text_processing/zh/test_time.py index 9a8e93f26..590fd591f 100644 --- a/tests/nemo_text_processing/zh/test_time.py +++ b/tests/nemo_text_processing/zh/test_time.py @@ -13,9 +13,10 @@ # limitations under the License. import pytest +from parameterized import parameterized + from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer from nemo_text_processing.text_normalization.normalize import Normalizer -from parameterized import parameterized from ..utils import CACHE_DIR, parse_test_case_file @@ -30,11 +31,11 @@ def test_norm_time(self, test_input, expected): preds = self.normalizer_zh.normalize(test_input) assert expected == preds - # inverse_normalizer = InverseNormalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False) + inverse_normalizer = InverseNormalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False) - # @parameterized.expand(parse_test_case_file('zh/data_inverse_text_normalization/test_cases_time.txt')) - # @pytest.mark.run_only_on('CPU') - # @pytest.mark.unit - # def test_denorm(self, test_input, expected): - # pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) - # assert pred == expected + @parameterized.expand(parse_test_case_file('zh/data_inverse_text_normalization/test_cases_time.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/zh/test_whitelist.py b/tests/nemo_text_processing/zh/test_whitelist.py new file mode 100644 index 000000000..deb857e7a --- /dev/null +++ b/tests/nemo_text_processing/zh/test_whitelist.py @@ -0,0 +1,41 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestDate: + normalizer_zh = Normalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('zh/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_date(self, test_input, expected): + preds = self.normalizer_zh.normalize(test_input) + assert expected == preds + + inverse_normalizer = InverseNormalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('zh/data_inverse_text_normalization/test_cases_date.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/zh/test_word.py b/tests/nemo_text_processing/zh/test_word.py index 29d396418..3314ea90b 100644 --- a/tests/nemo_text_processing/zh/test_word.py +++ b/tests/nemo_text_processing/zh/test_word.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,19 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. + import pytest -from nemo_text_processing.text_normalization.normalize import Normalizer from parameterized import parameterized +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer +from nemo_text_processing.text_normalization.normalize import Normalizer + from ..utils import CACHE_DIR, parse_test_case_file -class TestChar: +class TestWord: + inverse_normalizer = InverseNormalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False) + + @parameterized.expand(parse_test_case_file('zh/data_inverse_text_normalization/test_cases_word.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) + assert pred == expected + normalizer_zh = Normalizer(lang='zh', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') @parameterized.expand(parse_test_case_file('zh/data_text_normalization/test_cases_word.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_norm_char(self, test_input, expected): + def test_norm_date(self, test_input, expected): preds = self.normalizer_zh.normalize(test_input) assert expected == preds diff --git a/tools/text_processing_deployment/Dockerfile b/tools/text_processing_deployment/Dockerfile index 0fa7d855b..972bab75e 100644 --- a/tools/text_processing_deployment/Dockerfile +++ b/tools/text_processing_deployment/Dockerfile @@ -16,23 +16,25 @@ # Dockerfile for C++ (inverse) text normalization backend Sparrowhawk https://github.com/google/sparrowhawk # set base image (host OS) -FROM conda/miniconda3 +FROM continuumio/miniconda3:25.3.1-1 + # set the working directory in the container WORKDIR /workspace # install dependencies RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list +RUN apt-get update && apt-get upgrade -y && apt-get install -y --reinstall build-essential pkg-config git make wget RUN conda install conda-build -y -RUN apt-get update && apt-get install -y --reinstall build-essential pkg-config && apt-get upgrade -y && apt-get install -y git && apt-get install make +RUN conda install -c conda-forge thrax=1.3.4 -y RUN git clone https://github.com/google/re2 RUN cd re2 && git checkout tags/2022-02-01 && make && make install -RUN apt-get install build-essential -y && apt-get install wget -y RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz RUN tar xzvf protobuf-2.5.0.tar.gz RUN cd protobuf-2.5.0 && ./configure && make && make install && ldconfig -RUN conda install -c conda-forge thrax=1.3.4 -y -RUN git clone https://github.com/yzhang123/sparrowhawk.git -RUN cd sparrowhawk && git checkout test && apt-get install -y autoconf && bash autoreconf && ./configure && make && make install && ldconfig +RUN printf "# Conda lib path \n/opt/conda/lib" > /etc/ld.so.conf.d/conda.so.conf +ENV CPPFLAGS="-I/opt/conda/include" +ENV LDFLAGS="-L/opt/conda/lib" +RUN git clone https://github.com/anand-nv/sparrowhawk.git && cd sparrowhawk && git checkout nemo_tests && apt-get install -y autoconf && bash autoreconf && ./configure && make && make install && ldconfig RUN git clone https://github.com/kward/shunit2.git RUN echo "DONE" diff --git a/tools/text_processing_deployment/docker/launch.sh b/tools/text_processing_deployment/docker/launch.sh index 09d1bb523..debd98883 100644 --- a/tools/text_processing_deployment/docker/launch.sh +++ b/tools/text_processing_deployment/docker/launch.sh @@ -14,13 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -MODE=${1:-"export"} +# this script runs Sparrowhawk tests in a docker container "locally" (not in CI/CD pipeline) + +MODE=${1:-"interactive"} LANGUAGE=${2:-"en"} INPUT_CASE=${3:-"lower_cased"} +GRAMMARS=${4:-"tn_grammars"} # tn_grammars or itn_grammars SCRIPT_DIR=$(cd $(dirname $0); pwd) -: ${CLASSIFY_DIR:="$SCRIPT_DIR/../$LANGUAGE/classify"} -: ${VERBALIZE_DIR:="$SCRIPT_DIR/../$LANGUAGE/verbalize"} -: ${CMD:=${4:-"/bin/bash"}} +GRAMMAR_DIR=${5:-${SCRIPT_DIR}"/.."} +CONFIG=${LANGUAGE}_${GRAMMARS}_${INPUT_CASE} + +: ${CLASSIFY_DIR:="$GRAMMAR_DIR/${CONFIG}/classify"} +: ${VERBALIZE_DIR:="$GRAMMAR_DIR/${CONFIG}/verbalize"} +: ${CMD:=${6:-"/bin/bash"}} MOUNTS="" MOUNTS+=" -v $CLASSIFY_DIR:/workspace/sparrowhawk/documentation/grammars/en_toy/classify" @@ -44,11 +50,17 @@ elif [[ $MODE == "test_itn_grammars" ]]; then fi echo $MOUNTS -docker run -it --rm \ +# Use -it for interactive mode, -i only for test mode +if [[ $MODE == "interactive" ]]; then + DOCKER_FLAGS="-it" +else + DOCKER_FLAGS="-i" +fi +docker run $DOCKER_FLAGS -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 --rm \ --shm-size=4g \ --ulimit memlock=-1 \ --ulimit stack=67108864 \ $MOUNTS \ -v $SCRIPT_DIR/../../../tests/nemo_text_processing/:/workspace/tests/ \ -w $WORK_DIR \ - sparrowhawk:latest $CMD \ No newline at end of file + sparrowhawk:latest $CMD diff --git a/tools/text_processing_deployment/export_grammars.sh b/tools/text_processing_deployment/export_grammars.sh index b36ae3a03..017472ae9 100644 --- a/tools/text_processing_deployment/export_grammars.sh +++ b/tools/text_processing_deployment/export_grammars.sh @@ -32,11 +32,12 @@ GRAMMARS="itn_grammars" # tn_grammars INPUT_CASE="lower_cased" # cased -LANGUAGE="en" # language, {'en', 'es', 'de','zh'} supports both TN and ITN, {'pt', 'ru', 'fr', 'vi'} supports ITN only -MODE="export" +LANGUAGE="en" # language, {'en', 'es', 'de','zh'} supports both TN and ITN, {'pt', 'ru', 'fr', 'vi', 'mr'} supports ITN only +MODE="export" # default is one of {'export', 'interactive', 'test', 'ci'}. Default "export" OVERWRITE_CACHE="True" # Set to False to re-use .far files FORCE_REBUILD="False" # Set to True to re-build docker file -WHITELIST=None # Path to a whitelist file, if None the default will be used +WHITELIST="" # Path to a whitelist file, if None the default will be used +FAR_PATH=$(pwd) # Path where the grammars should be written for ARG in "$@" do @@ -50,7 +51,8 @@ do done -CACHE_DIR=${LANGUAGE} +CACHE_DIR=${FAR_PATH}/${LANGUAGE}_${GRAMMARS}_${INPUT_CASE} + echo "GRAMMARS = $GRAMMARS" echo "MODE = $MODE" echo "LANGUAGE = $LANGUAGE" @@ -60,12 +62,34 @@ echo "OVERWRITE_CACHE = $OVERWRITE_CACHE" echo "FORCE_REBUILD = $FORCE_REBUILD" echo "WHITELIST = $WHITELIST" +# check if WHITELIST file exists +if [[ ${WHITELIST} != "" ]] && [[ -f $WHITELIST ]]; then + WHITELIST="--whitelist=${WHITELIST} " + echo "[I] Whitelist file wasn't provided or doesn't exist, using default" +else + WHITELIST="" +fi + -if [[ ${OVERWRITE_CACHE,,} == "true" ]]; then +if [[ ${OVERWRITE_CACHE,,} == "true" ]] ; then OVERWRITE_CACHE="--overwrite_cache " - python3 pynini_export.py --output_dir=. --grammars=${GRAMMARS} --input_case=${INPUT_CASE} \ - --language=${LANGUAGE} --cache_dir=${CACHE_DIR} --whitelist=${WHITELIST} ${OVERWRITE_CACHE} || exit 1 - else OVERWRITE_CACHE="" +else + OVERWRITE_CACHE="" +fi + +CLASSIFY_FAR=${CACHE_DIR}"/classify/tokenize_and_classify.far" +VERBALIZE_FAR=${CACHE_DIR}"/verbalize/verbalize.far" + +# check if .far files do not exist +if [[ ! -f $CLASSIFY_FAR ]] || [[ ! -f $VERBALIZE_FAR ]] ; then + echo "[I] FSTs do not exist, will overwrite cache" + OVERWRITE_CACHE="--overwrite_cache " +fi + +if [[ ${OVERWRITE_CACHE} != "" ]] ; then + echo "[I] Exporting grammars" + python3 pynini_export.py --output_dir=${FAR_PATH} --grammars=${GRAMMARS} --input_case=${INPUT_CASE} \ + --language=${LANGUAGE} --cache_dir=${CACHE_DIR} ${WHITELIST} ${OVERWRITE_CACHE} || exit 1 fi if [[ ${FORCE_REBUILD,,} == "true" ]]; then @@ -74,10 +98,12 @@ if [[ ${FORCE_REBUILD,,} == "true" ]]; then fi find . -name "Makefile" -type f -delete -bash docker/build.sh $FORCE_REBUILD -if [[ ${MODE} == "test" ]]; then +if [[ ${MODE} == "test" ]] || [[ ${MODE} == "interactive" ]]; then MODE=${MODE}_${GRAMMARS} -fi - -bash docker/launch.sh $MODE $LANGUAGE $INPUT_CASE + bash docker/build.sh $FORCE_REBUILD + bash docker/launch.sh $MODE $LANGUAGE $INPUT_CASE $GRAMMARS $FAR_PATH +else + echo "done mode: $MODE" + exit 0 +fi \ No newline at end of file diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 6e87742ef..73a4fc138 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -1,4 +1,5 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, DIGITAL UMUGANDA # Copyright 2015 and onwards Google, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +20,7 @@ from argparse import ArgumentParser import pynini + from nemo_text_processing.text_normalization.en.graph_utils import generator_main # This script exports compiled grammars inside nemo_text_processing into OpenFst finite state archive files @@ -36,6 +38,8 @@ def itn_grammars(**kwargs): ).fst } d['verbalize'] = {'ALL': ITNVerbalizeFst().fst, 'REDUP': pynini.accep("REDUP")} + if ITNPostProcessingFst is not None: + d['post_process'] = {'POSTPROCESSOR': ITNPostProcessingFst().fst} return d @@ -51,6 +55,8 @@ def tn_grammars(**kwargs): ).fst } d['verbalize'] = {'ALL': TNVerbalizeFst(deterministic=True).fst, 'REDUP': pynini.accep("REDUP")} + if TNPostProcessingFst is not None: + d['post_process'] = {'POSTPROCESSOR': TNPostProcessingFst().fst} return d @@ -65,6 +71,8 @@ def export_grammars(output_dir, grammars): for category, graphs in grammars.items(): out_dir = os.path.join(output_dir, category) + if category == "post_process": + out_dir = os.path.join(output_dir, "verbalize") if not os.path.exists(out_dir): os.makedirs(out_dir) time.sleep(1) @@ -79,7 +87,29 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "pt", "ru", 'fr', 'hu', 'sv', 'vi', 'zh', 'ar', 'it', 'es_en'], + choices=[ + "en", + "de", + "es", + "pt", + "ru", + 'fr', + 'hu', + 'sv', + 'vi', + 'zh', + 'ar', + 'it', + 'es_en', + 'he', + 'hi', + 'hi_en', + 'hy', + 'mr', + 'ja', + 'rw', + 'ko', + ], type=str, default='en', ) @@ -110,9 +140,10 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - if args.language in ['pt', 'ru', 'vi', 'es_en'] and args.grammars == 'tn_grammars': + if args.language in ['ru', 'es_en', 'hi_en', 'mr'] and args.grammars == 'tn_grammars': raise ValueError('Only ITN grammars could be deployed in Sparrowhawk for the selected languages.') - + TNPostProcessingFst = None + ITNPostProcessingFst = None if args.language == 'en': from nemo_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst, @@ -123,7 +154,11 @@ def parse_args(): from nemo_text_processing.text_normalization.en.taggers.tokenize_and_classify import ( ClassifyFst as TNClassifyFst, ) + from nemo_text_processing.text_normalization.en.verbalizers.post_processing import ( + PostProcessingFst as TNPostProcessingFst, + ) from nemo_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'de': from nemo_text_processing.inverse_text_normalization.de.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst, @@ -160,6 +195,10 @@ def parse_args(): from nemo_text_processing.inverse_text_normalization.pt.verbalizers.verbalize import ( VerbalizeFst as ITNVerbalizeFst, ) + from nemo_text_processing.text_normalization.pt.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.pt.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'fr': from nemo_text_processing.inverse_text_normalization.fr.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst, @@ -171,6 +210,20 @@ def parse_args(): ClassifyFst as TNClassifyFst, ) from nemo_text_processing.text_normalization.fr.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'hi': + from nemo_text_processing.inverse_text_normalization.hi.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + from nemo_text_processing.text_normalization.hi.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.hi.verbalizers.post_processing import ( + PostProcessingFst as TNPostProcessingFst, + ) + from nemo_text_processing.text_normalization.hi.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'hu': from nemo_text_processing.text_normalization.hu.taggers.tokenize_and_classify import ( ClassifyFst as TNClassifyFst, @@ -194,6 +247,13 @@ def parse_args(): from nemo_text_processing.inverse_text_normalization.vi.verbalizers.verbalize import ( VerbalizeFst as ITNVerbalizeFst, ) + from nemo_text_processing.text_normalization.vi.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.vi.verbalizers.post_processing import ( + PostProcessingFst as TNPostProcessingFst, + ) + from nemo_text_processing.text_normalization.vi.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'zh': from nemo_text_processing.inverse_text_normalization.zh.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst, @@ -204,6 +264,9 @@ def parse_args(): from nemo_text_processing.text_normalization.zh.taggers.tokenize_and_classify import ( ClassifyFst as TNClassifyFst, ) + from nemo_text_processing.text_normalization.zh.verbalizers.post_processing import ( + PostProcessingFst as TNPostProcessingFst, + ) from nemo_text_processing.text_normalization.zh.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'ar': from nemo_text_processing.inverse_text_normalization.ar.taggers.tokenize_and_classify import ( @@ -215,6 +278,7 @@ def parse_args(): from nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ( ClassifyFst as TNClassifyFst, ) + from nemo_text_processing.text_normalization.ar.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'it': from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ( ClassifyFst as TNClassifyFst, @@ -227,7 +291,70 @@ def parse_args(): from nemo_text_processing.inverse_text_normalization.es_en.verbalizers.verbalize import ( VerbalizeFst as ITNVerbalizeFst, ) - output_dir = os.path.join(args.output_dir, args.language) + elif args.language == 'hi_en': + from nemo_text_processing.inverse_text_normalization.hi_en.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + elif args.language == 'mr': + from nemo_text_processing.inverse_text_normalization.mr.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.mr.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + elif args.language == 'he': + from nemo_text_processing.inverse_text_normalization.he.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.he.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + elif args.language == 'hy': + from nemo_text_processing.inverse_text_normalization.hy.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.hy.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + elif args.language == 'ja': + from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.ja.verbalizers.post_processing import ( + PostProcessingFst as ITNPostProcessingFst, + ) + from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + from nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.ja.verbalizers.post_processing import ( + PostProcessingFst as TNPostProcessingFst, + ) + from nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'rw': + from nemo_text_processing.text_normalization.rw.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.rw.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'ko': + from nemo_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) + from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.ko.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + else: + raise KeyError(f"Language {args.language} is not defined for export.") + output_dir = os.path.join(args.output_dir, f"{args.language}_{args.grammars}_{args.input_case}") export_grammars( output_dir=output_dir, grammars=locals()[args.grammars]( diff --git a/tools/text_processing_deployment/sh_test.sh b/tools/text_processing_deployment/sh_test.sh new file mode 100644 index 000000000..3bd1a2faa --- /dev/null +++ b/tools/text_processing_deployment/sh_test.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script runs the CI/CD tests for Sparrowhawk. It calls export_grammars.sh to create the grammars. + + +GRAMMARS="itn_grammars" # tn_grammars +INPUT_CASE="lower_cased" # cased +LANGUAGE="en" # language, {'en', 'es', 'de','zh', 'pt'} supports both TN and ITN, {'ru', 'fr', 'vi', 'pt'} supports ITN only +OVERWRITE_CACHE="False" # Set to False to re-use .far files +WHITELIST="" # Path to a whitelist file, if None the default will be used +FAR_PATH=$(pwd) # Path where the grammars should be written +MODE="test_itn_grammars" + +for ARG in "$@" +do + key=$(echo $ARG | cut -f1 -d=) + value=$(echo $ARG | cut -f2 -d=) + + if [[ $key == *"--"* ]]; then + v="${key/--/}" + declare $v="${value}" + fi +done + + +CACHE_DIR=${FAR_PATH}/${LANGUAGE} +echo "GRAMMARS = $GRAMMARS" +echo "LANGUAGE = $LANGUAGE" +echo "INPUT_CASE = $INPUT_CASE" +echo "CACHE_DIR = $CACHE_DIR" +echo "OVERWRITE_CACHE = $OVERWRITE_CACHE" +echo "FORCE_REBUILD = $FORCE_REBUILD" +echo "WHITELIST = $WHITELIST" + +if [[ ${WHITELIST} != "" ]] && [[ -f $WHITELIST ]]; then + WHITELIST="--whitelist=${WHITELIST} " + echo "[I] Whitelist file wasn't provided or doesn't exist, using default" +else + WHITELIST="" +fi + +bash export_grammars.sh --MODE="export" --GRAMMARS=$GRAMMARS --LANGUAGE=$LANGUAGE --INPUT_CASE=$INPUT_CASE \ + --FAR_PATH=$FAR_PATH --CACHE_DIR=$CACHE_DIR --OVERWRITE_CACHE=$OVERWRITE_CACHE \ + --FORCE_REBUILD=$FORCE_REBUILD $WHITELIST + +CLASSIFY_FAR=${CACHE_DIR}_${GRAMMARS}_${INPUT_CASE}/classify/tokenize_and_classify.far +VERBALIZE_FAR=${CACHE_DIR}_${GRAMMARS}_${INPUT_CASE}/verbalize/verbalize.far + +CONFIG=${LANGUAGE}_${GRAMMARS}_${INPUT_CASE} + +cp $CLASSIFY_FAR /workspace/sparrowhawk/documentation/grammars_${CONFIG}/en_toy/classify/ +cp $VERBALIZE_FAR /workspace/sparrowhawk/documentation/grammars_${CONFIG}/en_toy/verbalize/ \ No newline at end of file diff --git a/tutorials/Text_(Inverse)_Normalization.ipynb b/tutorials/Text_(Inverse)_Normalization.ipynb index 9aea8317c..6a21e3856 100644 --- a/tutorials/Text_(Inverse)_Normalization.ipynb +++ b/tutorials/Text_(Inverse)_Normalization.ipynb @@ -92,6 +92,7 @@ "source": [ "# create text normalization instance that works on cased input\n", "from nemo_text_processing.text_normalization.normalize import Normalizer\n", + "\n", "normalizer = Normalizer(input_case='cased', lang='en')" ] }, @@ -101,7 +102,7 @@ "metadata": {}, "outputs": [], "source": [ - "# the normalizer class offers the following parameterization. \n", + "# the normalizer class offers the following parameterization.\n", "print(normalizer.__doc__)" ] }, @@ -246,7 +247,7 @@ "outputs": [], "source": [ "# example evaluation sentence\n", - "eval_text = \"\"\"PLAIN\\ton\\t\n", + "eval_text = \"\"\"PLAIN\\ton\\t\n", "DATE\\t22 july 2012\\tthe twenty second of july twenty twelve\n", "PLAIN\\tthey\\t\n", "PLAIN\\tworked\\t\n", @@ -278,6 +279,7 @@ "source": [ "# Parse evaluation file into written and normalized sentence pairs\n", "from nemo_text_processing.text_normalization.data_loader_utils import load_files, training_data_to_sentences\n", + "\n", "eval_data = load_files([EVAL_FILE])\n", "sentences_un_normalized, sentences_normalized, sentences_class_types = training_data_to_sentences(eval_data)\n", "print(list(zip(sentences_un_normalized, sentences_normalized)))" @@ -302,9 +304,8 @@ "source": [ "# measure sentence accuracy\n", "from nemo_text_processing.text_normalization.data_loader_utils import evaluate\n", - "sentences_accuracy = evaluate(\n", - " preds=sentences_prediction, labels=sentences_normalized, input=sentences_un_normalized\n", - " )\n", + "\n", + "sentences_accuracy = evaluate(preds=sentences_prediction, labels=sentences_normalized, input=sentences_un_normalized)\n", "print(\"- Accuracy: \" + str(sentences_accuracy))" ] }, @@ -324,6 +325,7 @@ "source": [ "# create inverse text normalization instance\n", "from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer\n", + "\n", "inverse_normalizer = InverseNormalizer(lang='en')" ] }, @@ -381,14 +383,14 @@ "metadata": {}, "outputs": [], "source": [ - "# initialize normalizer, this may take some time to generate the extended grammars. \n", + "# initialize normalizer, this may take some time to generate the extended grammars.\n", "# Thus, we recommend to cache the grammars by specifying a cache directory\n", "normalizer = NormalizerWithAudio(\n", - " lang=\"en\",\n", - " input_case=\"cased\",\n", - " overwrite_cache=False,\n", - " cache_dir=\"cache_dir\",\n", - " )\n", + " lang=\"en\",\n", + " input_case=\"cased\",\n", + " overwrite_cache=False,\n", + " cache_dir=\"cache_dir\",\n", + ")\n", "# create up to 10 normalization options\n", "print(normalizer.normalize(\"123\", n_tagged=10, punct_post_process=True))" ] diff --git a/tutorials/WFST_Tutorial.ipynb b/tutorials/WFST_Tutorial.ipynb index 5b1c0530e..2963e905d 100644 --- a/tutorials/WFST_Tutorial.ipynb +++ b/tutorials/WFST_Tutorial.ipynb @@ -60,7 +60,15 @@ "metadata": {}, "outputs": [], "source": [ - "from nemo_text_processing.text_normalization.en.graph_utils import GraphFst, NEMO_DIGIT, delete_space, NEMO_SIGMA, NEMO_NOT_QUOTE, delete_extra_space, NEMO_NON_BREAKING_SPACE\n", + "from nemo_text_processing.text_normalization.en.graph_utils import (\n", + " GraphFst,\n", + " NEMO_DIGIT,\n", + " delete_space,\n", + " NEMO_SIGMA,\n", + " NEMO_NOT_QUOTE,\n", + " delete_extra_space,\n", + " NEMO_NON_BREAKING_SPACE,\n", + ")\n", "from nemo_text_processing.text_normalization.normalize import Normalizer\n", "\n", "from nemo_text_processing.inverse_text_normalization.fr.taggers.cardinal import CardinalFst\n", @@ -77,7 +85,7 @@ "from nemo_text_processing.inverse_text_normalization.fr.verbalizers.ordinal import OrdinalFst\n", "from nemo_text_processing.inverse_text_normalization.fr.verbalizers.time import TimeFst\n", "from nemo_text_processing.inverse_text_normalization.fr.verbalizers.whitelist import WhiteListFst\n", - "from nemo_text_processing.inverse_text_normalization.fr.verbalizers.word import WordFst\n" + "from nemo_text_processing.inverse_text_normalization.fr.verbalizers.word import WordFst" ] }, { @@ -492,16 +500,17 @@ "source": [ "from pynini.lib import pynutil\n", "\n", + "\n", "def apply_fst(text, fst):\n", - " \"\"\" Given a string input, returns the output string\n", - " produced by traversing the path with lowest weight.\n", - " If no valid path accepts input string, returns an\n", - " error.\n", - " \"\"\"\n", - " try:\n", - " print(pynini.shortestpath(text @ fst).string())\n", - " except pynini.FstOpError:\n", - " print(f\"Error: No valid output with given input: '{text}'\")" + " \"\"\"Given a string input, returns the output string\n", + " produced by traversing the path with lowest weight.\n", + " If no valid path accepts input string, returns an\n", + " error.\n", + " \"\"\"\n", + " try:\n", + " print(pynini.shortestpath(text @ fst).string())\n", + " except pynini.FstOpError:\n", + " print(f\"Error: No valid output with given input: '{text}'\")" ] }, { @@ -563,19 +572,21 @@ }, "outputs": [], "source": [ - "zero = pynini.string_map([(\"zéro\",\"0\")]) # French only pronounces zeroes as stand alone\n", - "digits = pynini.string_map([ # pynini function that creates explicit input-output mappings for a WFST\n", - "\t\t\t\t(\"un\",\"1\"),\n", - "\t\t\t\t(\"une\",\"1\"),\n", - "\t\t\t\t(\"deux\",\"2\"),\n", - "\t\t\t\t(\"trois\",\"3\"),\n", - "\t\t\t\t(\"quatre\",\"4\"),\n", - "\t\t\t\t(\"cinq\",\"5\"),\n", - "\t\t\t\t(\"six\",\"6\"),\n", - "\t\t\t\t(\"sept\",\"7\"),\n", - "\t\t\t\t(\"huit\",\"8\"),\n", - "\t\t\t\t(\"neuf\",\"9\")\n", - "])" + "zero = pynini.string_map([(\"zéro\", \"0\")]) # French only pronounces zeroes as stand alone\n", + "digits = pynini.string_map(\n", + " [ # pynini function that creates explicit input-output mappings for a WFST\n", + " (\"un\", \"1\"),\n", + " (\"une\", \"1\"),\n", + " (\"deux\", \"2\"),\n", + " (\"trois\", \"3\"),\n", + " (\"quatre\", \"4\"),\n", + " (\"cinq\", \"5\"),\n", + " (\"six\", \"6\"),\n", + " (\"sept\", \"7\"),\n", + " (\"huit\", \"8\"),\n", + " (\"neuf\", \"9\"),\n", + " ]\n", + ")" ] }, { @@ -678,14 +689,16 @@ }, "outputs": [], "source": [ - "teens = pynini.string_map([\n", - "\t\t\t\t(\"onze\",\"11\"),\n", - "\t\t\t\t(\"douze\",\"12\"),\n", - "\t\t\t\t(\"treize\",\"13\"),\n", - "\t\t\t\t(\"quatorze\",\"14\"),\n", - "\t\t\t\t(\"quinze\",\"15\"),\n", - "\t\t\t\t(\"seize\",\"16\"),\n", - "])" + "teens = pynini.string_map(\n", + " [\n", + " (\"onze\", \"11\"),\n", + " (\"douze\", \"12\"),\n", + " (\"treize\", \"13\"),\n", + " (\"quatorze\", \"14\"),\n", + " (\"quinze\", \"15\"),\n", + " (\"seize\", \"16\"),\n", + " ]\n", + ")" ] }, { @@ -706,7 +719,9 @@ "outputs": [], "source": [ "tens = pynini.string_map([(\"dix\", \"1\")])\n", - "delete_hyphen = pynini.closure(pynutil.delete(\"-\"), 0, 1) # Applies a closure from 0-1 of operation. Equivalent to regex /?/\n", + "delete_hyphen = pynini.closure(\n", + " pynutil.delete(\"-\"), 0, 1\n", + ") # Applies a closure from 0-1 of operation. Equivalent to regex /?/\n", "\n", "graph_tens = tens + delete_hyphen + digits" ] @@ -820,7 +835,7 @@ }, "outputs": [], "source": [ - "graph_digits = digits | pynutil.insert(\"0\") # inserts zero if no digit follows" + "graph_digits = digits | pynutil.insert(\"0\") # inserts zero if no digit follows" ] }, { @@ -862,7 +877,7 @@ "outputs": [], "source": [ "graph_teens_and_tens = graph_tens | teens\n", - "graph_all = graph_teens_and_tens | zero " + "graph_all = graph_teens_and_tens | zero" ] }, { @@ -886,13 +901,13 @@ }, "outputs": [], "source": [ - "apply_fst(\"dix-huit\", graph_all) \n", - "apply_fst(\"seize\" , graph_all)\n", - "apply_fst(\"dix\" , graph_all) \n", - "apply_fst(\"une\" , graph_all) \n", - "apply_fst(\"trois\" , graph_all) \n", - "apply_fst(\"quatre\" , graph_all) \n", - "apply_fst(\"zéro\" , graph_all)" + "apply_fst(\"dix-huit\", graph_all)\n", + "apply_fst(\"seize\", graph_all)\n", + "apply_fst(\"dix\", graph_all)\n", + "apply_fst(\"une\", graph_all)\n", + "apply_fst(\"trois\", graph_all)\n", + "apply_fst(\"quatre\", graph_all)\n", + "apply_fst(\"zéro\", graph_all)" ] }, { @@ -959,14 +974,16 @@ }, "outputs": [], "source": [ - "tens = pynini.string_map([\n", - "\t\t\t\t(\"dix\", \"1\"),\n", - "\t\t\t\t(\"vingt\",\"2\"),\n", - "\t\t\t\t(\"trente\",\"3\"),\n", - "\t\t\t\t(\"quarante\",\"4\"),\n", - "\t\t\t\t(\"cinquante\",\"5\"),\n", - "\t\t\t\t(\"soixante\",\"6\"),\n", - "\t\t])\n", + "tens = pynini.string_map(\n", + " [\n", + " (\"dix\", \"1\"),\n", + " (\"vingt\", \"2\"),\n", + " (\"trente\", \"3\"),\n", + " (\"quarante\", \"4\"),\n", + " (\"cinquante\", \"5\"),\n", + " (\"soixante\", \"6\"),\n", + " ]\n", + ")\n", "\n", "graph_et = pynutil.delete(\"-et-\")\n", "\n", @@ -974,7 +991,7 @@ "\n", "graph_tens = tens + graph_digits\n", "graph_teens_and_tens = graph_tens | teens\n", - "graph_all = graph_teens_and_tens | zero " + "graph_all = graph_teens_and_tens | zero" ] }, { @@ -1045,19 +1062,21 @@ }, "outputs": [], "source": [ - "tens = pynini.string_map([\n", - "\t\t\t\t(\"dix\", \"1\"),\n", - "\t\t\t\t(\"vingt\",\"2\"),\n", - "\t\t\t\t(\"trente\",\"3\"),\n", - "\t\t\t\t(\"quarante\",\"4\"),\n", - "\t\t\t\t(\"cinquante\",\"5\"),\n", - "\t\t\t\t(\"soixante\",\"6\"),\n", - " (\"quatre-vingt\", \"8\")\n", - "\t\t])\n", + "tens = pynini.string_map(\n", + " [\n", + " (\"dix\", \"1\"),\n", + " (\"vingt\", \"2\"),\n", + " (\"trente\", \"3\"),\n", + " (\"quarante\", \"4\"),\n", + " (\"cinquante\", \"5\"),\n", + " (\"soixante\", \"6\"),\n", + " (\"quatre-vingt\", \"8\"),\n", + " ]\n", + ")\n", "tens = tens | pynutil.insert(\"0\") | tens + delete_hyphen | tens + graph_et\n", "graph_tens = tens + graph_digits\n", "graph_teens_and_tens = graph_tens | teens\n", - "graph_all = graph_teens_and_tens | zero " + "graph_all = graph_teens_and_tens | zero" ] }, { @@ -1121,9 +1140,7 @@ }, "outputs": [], "source": [ - "quatre_vingt_plural = pynini.string_map([\n", - " (\"quatre-vingts\", \"80\")\n", - "\t\t])" + "quatre_vingt_plural = pynini.string_map([(\"quatre-vingts\", \"80\")])" ] }, { @@ -1143,21 +1160,21 @@ }, "outputs": [], "source": [ - "quatre_vingt_singular = pynini.string_map([\n", - " (\"quatre-vingt-\", \"8\") # Note that the hyphen can be assumed now\n", - "\t\t])\n", - "graph_digits_without_zero = pynini.string_map([\n", - "\t\t\t\t(\"un\",\"1\"),\n", - "\t\t\t\t(\"une\",\"1\"),\n", - "\t\t\t\t(\"deux\",\"2\"),\n", - "\t\t\t\t(\"trois\",\"3\"),\n", - "\t\t\t\t(\"quatre\",\"4\"),\n", - "\t\t\t\t(\"cinq\",\"5\"),\n", - "\t\t\t\t(\"six\",\"6\"),\n", - "\t\t\t\t(\"sept\",\"7\"),\n", - "\t\t\t\t(\"huit\",\"8\"),\n", - "\t\t\t\t(\"neuf\",\"9\")\n", - "])\n", + "quatre_vingt_singular = pynini.string_map([(\"quatre-vingt-\", \"8\")]) # Note that the hyphen can be assumed now\n", + "graph_digits_without_zero = pynini.string_map(\n", + " [\n", + " (\"un\", \"1\"),\n", + " (\"une\", \"1\"),\n", + " (\"deux\", \"2\"),\n", + " (\"trois\", \"3\"),\n", + " (\"quatre\", \"4\"),\n", + " (\"cinq\", \"5\"),\n", + " (\"six\", \"6\"),\n", + " (\"sept\", \"7\"),\n", + " (\"huit\", \"8\"),\n", + " (\"neuf\", \"9\"),\n", + " ]\n", + ")\n", "graph_eighties = (quatre_vingt_singular + graph_digits_without_zero) | quatre_vingt_plural" ] }, @@ -1178,33 +1195,43 @@ }, "outputs": [], "source": [ - "seventy_and_ninety = pynini.string_map([\n", + "seventy_and_ninety = pynini.string_map(\n", + " [\n", " (\"soixante-dix\", \"70\"),\n", " (\"quatre-vingt-dix\", \"90\"),\n", - "\t\t])\n", + " ]\n", + ")\n", "\n", - "seventy_and_ninety_tens = pynini.string_map([\n", + "seventy_and_ninety_tens = pynini.string_map(\n", + " [\n", " (\"soixante-\", \"7\"),\n", " (\"quatre-vingt-\", \"9\"),\n", - "\t\t])\n", + " ]\n", + ")\n", "\n", - "seventy_and_ninety_one = pynini.string_map([\n", + "seventy_and_ninety_one = pynini.string_map(\n", + " [\n", " (\"soixante-et-onze\", \"71\"),\n", " (\"quatre-vingt-onze\", \"91\"),\n", - "\t\t])\n", + " ]\n", + ")\n", "\n", - "seventy_and_ninety_digits = digits = pynini.string_map([ \n", - "\t\t\t\t(\"douze\",\"2\"),\n", - "\t\t\t\t(\"treize\",\"3\"),\n", - "\t\t\t\t(\"quatorze\",\"4\"),\n", - "\t\t\t\t(\"quinze\",\"5\"),\n", - "\t\t\t\t(\"seize\",\"6\"),\n", - "\t\t\t\t(\"dix-sept\",\"7\"), # For 97-99, digits are used as normal.\n", - "\t\t\t\t(\"dix-huit\",\"8\"),\n", - "\t\t\t\t(\"dix-neuf\",\"9\")\n", - "])\n", + "seventy_and_ninety_digits = digits = pynini.string_map(\n", + " [\n", + " (\"douze\", \"2\"),\n", + " (\"treize\", \"3\"),\n", + " (\"quatorze\", \"4\"),\n", + " (\"quinze\", \"5\"),\n", + " (\"seize\", \"6\"),\n", + " (\"dix-sept\", \"7\"), # For 97-99, digits are used as normal.\n", + " (\"dix-huit\", \"8\"),\n", + " (\"dix-neuf\", \"9\"),\n", + " ]\n", + ")\n", "\n", - "graph_seventies_and_nineties = (seventy_and_ninety_tens + seventy_and_ninety_digits) | seventy_and_ninety | seventy_and_ninety_one " + "graph_seventies_and_nineties = (\n", + " (seventy_and_ninety_tens + seventy_and_ninety_digits) | seventy_and_ninety | seventy_and_ninety_one\n", + ")" ] }, { @@ -1224,20 +1251,22 @@ }, "outputs": [], "source": [ - "tens = pynini.string_map([\n", - "\t\t\t\t(\"dix\", \"1\"),\n", - "\t\t\t\t(\"vingt\",\"2\"),\n", - "\t\t\t\t(\"trente\",\"3\"),\n", - "\t\t\t\t(\"quarante\",\"4\"),\n", - "\t\t\t\t(\"cinquante\",\"5\"),\n", - "\t\t\t\t(\"soixante\",\"6\"),\n", - "\t\t])\n", + "tens = pynini.string_map(\n", + " [\n", + " (\"dix\", \"1\"),\n", + " (\"vingt\", \"2\"),\n", + " (\"trente\", \"3\"),\n", + " (\"quarante\", \"4\"),\n", + " (\"cinquante\", \"5\"),\n", + " (\"soixante\", \"6\"),\n", + " ]\n", + ")\n", "tens = tens | pynutil.insert(\"0\") | tens + delete_hyphen | tens + graph_et\n", "\n", "graph_tens = tens + graph_digits\n", "graph_tens_with_special_cases = graph_tens | graph_seventies_and_nineties | graph_eighties\n", "graph_teens_and_tens = graph_tens_with_special_cases | teens\n", - "graph_all = graph_teens_and_tens | zero " + "graph_all = graph_teens_and_tens | zero" ] }, { @@ -1261,7 +1290,7 @@ }, "outputs": [], "source": [ - "apply_fst(\"quatre-vingt-treize\" , graph_all)\n", + "apply_fst(\"quatre-vingt-treize\", graph_all)\n", "apply_fst(\"quatre-vingts\", graph_all)\n", "apply_fst(\"quatre-vingt-deux\", graph_all)" ] @@ -1314,9 +1343,9 @@ "outputs": [], "source": [ "apply_fst(\"dix-une\", graph_all) # supposed to be \"onze\"\n", - "apply_fst(\"dix-deux\", graph_all) # supposed to be \"douze\"\n", + "apply_fst(\"dix-deux\", graph_all) # supposed to be \"douze\"\n", "apply_fst(\"vingt-un\", graph_all) # supposed to be \"vingt-et-un\"\n", - "apply_fst(\"trente-un\", graph_all) # supposed to be \"trente-et-un\"" + "apply_fst(\"trente-un\", graph_all) # supposed to be \"trente-et-un\"" ] }, { @@ -1340,38 +1369,41 @@ }, "outputs": [], "source": [ - "graph_tens_special = pynini.string_map([\n", - "\t\t\t\t(\"soixante-dix\", \"70\"),\n", - "\t\t\t\t(\"soixante-et-onze\",\"71\"),\n", - " (\"soixante-douze\",\"72\"),\n", - "\t\t\t\t(\"soixante-treize\",\"73\"),\n", - "\t\t\t\t(\"soizante-quatorze\",\"74\"),\n", - "\t\t\t\t(\"soixante-quinze\",\"75\"),\n", - "\t\t\t\t(\"soixante-seize\",\"76\"),\n", - " (\"soixante-dix-sept\",\"77\"),\n", - " (\"soixante-dix-huit\",\"78\"),\n", - "\t\t\t\t(\"soixante-dix-neuf\",\"79\"),\n", + "graph_tens_special = pynini.string_map(\n", + " [\n", + " (\"soixante-dix\", \"70\"),\n", + " (\"soixante-et-onze\", \"71\"),\n", + " (\"soixante-douze\", \"72\"),\n", + " (\"soixante-treize\", \"73\"),\n", + " (\"soizante-quatorze\", \"74\"),\n", + " (\"soixante-quinze\", \"75\"),\n", + " (\"soixante-seize\", \"76\"),\n", + " (\"soixante-dix-sept\", \"77\"),\n", + " (\"soixante-dix-huit\", \"78\"),\n", + " (\"soixante-dix-neuf\", \"79\"),\n", " (\"quatre-vingts\", \"80\"),\n", " (\"quatre-vingt-un\", \"81\"),\n", " (\"quatre-vingt-une\", \"81\"),\n", - "\t\t\t\t(\"quatre-vingt-deux\",\"82\"),\n", - " (\"quatre-vingt-trois\",\"83\"),\n", - " (\"quatre-vingt-quatre\",\"84\"),\n", - " (\"quatre-vingt-cinq\",\"85\"),\n", - " (\"quatre-vingt-six\",\"86\"),\n", - " (\"quatre-vingt-sept\",\"87\"),\n", - " (\"quatre-vingt-huit\",\"88\"),\n", - " (\"quatre-vingt-neuf\",\"89\"),\n", - " (\"quatre-vingt-dix\",\"90\"),\n", - " (\"quatre-vingt-onze\",\"91\"),\n", - " (\"quatre-vingt-douze\",\"92\"),\n", - " (\"quatre-vingt-treize\",\"93\"),\n", - " (\"quatre-vingt-quatorze\",\"94\"),\n", - " (\"quatre-vingt-quinze\",\"95\"),\n", - " (\"quatre-vingt-sieze\",\"96\"),\n", - " (\"quatre-vingt-dix-sept\",\"97\"),\n", - " (\"quatre-vingt-dix-huit\",\"98\"),\n", - " (\"quatre-vingt-dix-neuf\",\"99\"),])" + " (\"quatre-vingt-deux\", \"82\"),\n", + " (\"quatre-vingt-trois\", \"83\"),\n", + " (\"quatre-vingt-quatre\", \"84\"),\n", + " (\"quatre-vingt-cinq\", \"85\"),\n", + " (\"quatre-vingt-six\", \"86\"),\n", + " (\"quatre-vingt-sept\", \"87\"),\n", + " (\"quatre-vingt-huit\", \"88\"),\n", + " (\"quatre-vingt-neuf\", \"89\"),\n", + " (\"quatre-vingt-dix\", \"90\"),\n", + " (\"quatre-vingt-onze\", \"91\"),\n", + " (\"quatre-vingt-douze\", \"92\"),\n", + " (\"quatre-vingt-treize\", \"93\"),\n", + " (\"quatre-vingt-quatorze\", \"94\"),\n", + " (\"quatre-vingt-quinze\", \"95\"),\n", + " (\"quatre-vingt-sieze\", \"96\"),\n", + " (\"quatre-vingt-dix-sept\", \"97\"),\n", + " (\"quatre-vingt-dix-huit\", \"98\"),\n", + " (\"quatre-vingt-dix-neuf\", \"99\"),\n", + " ]\n", + ")" ] }, { @@ -1395,7 +1427,7 @@ }, "outputs": [], "source": [ - "constructed_version = (graph_seventies_and_nineties | graph_eighties)\n", + "constructed_version = graph_seventies_and_nineties | graph_eighties\n", "constructed_version.num_states()" ] }, @@ -1535,7 +1567,7 @@ "source": [ "apply_fst(\"deux-cent-trois\", hundreds)\n", "apply_fst(\"huit-cent-quatre-vingts\", hundreds)\n", - "apply_fst(\"cinq-cent-trente\" , hundreds) " + "apply_fst(\"cinq-cent-trente\", hundreds)" ] }, { @@ -1559,7 +1591,9 @@ }, "outputs": [], "source": [ - "cents = pynini.accep(\"cent\") | pynini.accep(\"cents\") # Creates a Finite State (Accep)tor, mapping inputs back to themselves\n", + "cents = pynini.accep(\"cent\") | pynini.accep(\n", + " \"cents\"\n", + ") # Creates a Finite State (Accep)tor, mapping inputs back to themselves\n", "hundreds = graph_digits + delete_hyphen + pynutil.delete(cents) + delete_hyphen + graph_all" ] }, @@ -1580,7 +1614,7 @@ }, "outputs": [], "source": [ - "graph_cents = pynini.cross(\"cents\", \"00\") # Creates a single input-output mapping\n", + "graph_cents = pynini.cross(\"cents\", \"00\") # Creates a single input-output mapping\n", "hundreds = graph_digits + delete_hyphen + ((pynutil.delete(\"cent\") + delete_hyphen + graph_all) | graph_cents)" ] }, @@ -1618,9 +1652,9 @@ }, "outputs": [], "source": [ - "apply_fst(\"trois-cents\", graph_hundreds) \n", + "apply_fst(\"trois-cents\", graph_hundreds)\n", "apply_fst(\"cent\", graph_hundreds)\n", - "apply_fst(\"cent-trois\", graph_hundreds) " + "apply_fst(\"cent-trois\", graph_hundreds)" ] }, { @@ -1738,8 +1772,8 @@ }, "outputs": [], "source": [ - "apply_fst(\"mille-cent-un\", graph_thousands) # Should be 1101\n", - "apply_fst(\"mille-cent\", graph_thousands) # 1100" + "apply_fst(\"mille-cent-un\", graph_thousands) # Should be 1101\n", + "apply_fst(\"mille-cent\", graph_thousands) # 1100" ] }, { @@ -1761,8 +1795,8 @@ "source": [ "graph_cents = pynini.cross(\"cents\", \"00\")\n", "graph_cent = pynini.cross(\"cent\", \"1\")\n", - "graph_hundreds_first_digit = (graph_digits + delete_hyphen + pynutil.delete(cents)) | graph_cent\n", - "graph_hundreds = (graph_hundreds_first_digit + delete_hyphen | pynutil.insert(\"0\")) + graph_all \n", + "graph_hundreds_first_digit = (graph_digits + delete_hyphen + pynutil.delete(cents)) | graph_cent\n", + "graph_hundreds = (graph_hundreds_first_digit + delete_hyphen | pynutil.insert(\"0\")) + graph_all\n", "\n", "graph_one_thousand = pynini.cross(\"mille\", \"1\")\n", "graph_many_thousand = graph_hundreds + delete_hyphen + pynutil.delete(\"mille\")\n", @@ -1788,7 +1822,7 @@ }, "outputs": [], "source": [ - "graph_hundreds = (graph_hundreds_first_digit + delete_hyphen | pynutil.insert(\"0\", weight=.1)) + graph_all \n", + "graph_hundreds = (graph_hundreds_first_digit + delete_hyphen | pynutil.insert(\"0\", weight=0.1)) + graph_all\n", "\n", "graph_one_thousand = pynini.cross(\"mille\", \"1\")\n", "graph_many_thousand = graph_hundreds + delete_hyphen + pynutil.delete(\"mille\")\n", @@ -1850,7 +1884,9 @@ "source": [ "graph_one_thousand = pynini.cross(\"mille\", \"1\")\n", "graph_many_thousand = graph_hundreds + delete_hyphen + pynutil.delete(\"mille\")\n", - "graph_thousands = (graph_one_thousand | graph_many_thousand | pynutil.insert(\"000\", weight=.001)) + delete_hyphen + graph_hundreds" + "graph_thousands = (\n", + " (graph_one_thousand | graph_many_thousand | pynutil.insert(\"000\", weight=0.001)) + delete_hyphen + graph_hundreds\n", + ")" ] }, { @@ -1887,8 +1923,10 @@ "outputs": [], "source": [ "millions = pynini.accep(\"million\") | pynini.accep(\"millions\")\n", - "graph_millions = ((graph_hundreds + delete_hyphen + pynutil.delete(millions) + delete_hyphen) | pynutil.insert(\"000\", weight=.1) # We need three zeroes now\n", - " ) + graph_thousands" + "graph_millions = (\n", + " (graph_hundreds + delete_hyphen + pynutil.delete(millions) + delete_hyphen)\n", + " | pynutil.insert(\"000\", weight=0.1) # We need three zeroes now\n", + ") + graph_thousands" ] }, { @@ -1900,8 +1938,10 @@ "outputs": [], "source": [ "billions = pynini.accep(\"milliards\") | pynini.accep(\"milliard\")\n", - "graph_billions = ((graph_hundreds + delete_hyphen + pynutil.delete(billions) + delete_hyphen)| pynutil.insert(\"000\",weight=.1) # We need three zeroes now\n", - " ) + graph_millions" + "graph_billions = (\n", + " (graph_hundreds + delete_hyphen + pynutil.delete(billions) + delete_hyphen)\n", + " | pynutil.insert(\"000\", weight=0.1) # We need three zeroes now\n", + ") + graph_millions" ] }, { @@ -1913,8 +1953,10 @@ "outputs": [], "source": [ "trillions = pynini.accep(\"billion\") | pynini.accep(\"billions\")\n", - "graph_trillions = ((graph_hundreds + delete_hyphen + pynutil.delete(trillions) + delete_hyphen) | pynutil.insert(\"000\",weight=.1) # We need three zeroes now\n", - " ) + graph_billions" + "graph_trillions = (\n", + " (graph_hundreds + delete_hyphen + pynutil.delete(trillions) + delete_hyphen)\n", + " | pynutil.insert(\"000\", weight=0.1) # We need three zeroes now\n", + ") + graph_billions" ] }, { @@ -1959,7 +2001,7 @@ "outputs": [], "source": [ "example = \"deux-cent-milliard-quatre-million-deux-cent-quatre-vingt-onze\"\n", - "apply_fst(example, graph) " + "apply_fst(example, graph)" ] }, { @@ -1993,14 +2035,18 @@ }, "outputs": [], "source": [ - "delete_leading_zeroes = pynutil.delete(pynini.closure(\"0\")) # will delete all zeroes under closure. Equivalent to regex * operator\n", - "stop_at_non_zero = pynini.difference(NEMO_DIGIT, \"0\") # creates a graph that accepts all input-outputs from NEMO_DIGIT except 0\n", - "rest_of_cardinal = pynini.closure(NEMO_DIGIT) # accepts all digits that may follow\n", + "delete_leading_zeroes = pynutil.delete(\n", + " pynini.closure(\"0\")\n", + ") # will delete all zeroes under closure. Equivalent to regex * operator\n", + "stop_at_non_zero = pynini.difference(\n", + " NEMO_DIGIT, \"0\"\n", + ") # creates a graph that accepts all input-outputs from NEMO_DIGIT except 0\n", + "rest_of_cardinal = pynini.closure(NEMO_DIGIT) # accepts all digits that may follow\n", "\n", "clean_cardinal = delete_leading_zeroes + stop_at_non_zero + rest_of_cardinal\n", - "clean_cardinal = clean_cardinal | \"0\" # We don't want to ignore the occurrence of zero\n", + "clean_cardinal = clean_cardinal | \"0\" # We don't want to ignore the occurrence of zero\n", "\n", - "graph = graph @ clean_cardinal " + "graph = graph @ clean_cardinal" ] }, { @@ -2103,8 +2149,8 @@ " def __init__(self):\n", " super().__init__(name=\"cardinal\", kind=\"classify\")\n", " # Rest of the grammar here\n", - " # ....... \n", - " #........." + " # .......\n", + " # ........." ] }, { @@ -2133,8 +2179,8 @@ " def __init__(self):\n", " super().__init__(name=\"cardinal\", kind=\"classify\")\n", " # Rest of the grammar here\n", - " # ....... \n", - " #.........\n", + " # .......\n", + " # .........\n", " self.fst = pynutil.insert(\"integer: \\\"\") + graph + pynutil.insert(\"\\\"\")" ] }, @@ -2159,8 +2205,8 @@ " def __init__(self):\n", " super().__init__(name=\"cardinal\", kind=\"classify\")\n", " # Rest of the grammar here\n", - " # ....... \n", - " #.........\n", + " # .......\n", + " # .........\n", " self.fst = pynutil.insert(\"integer: \\\"\") + graph + pynutil.insert(\"\\\"\")\n", " final_graph = self.add_tokens(graph)" ] @@ -2187,7 +2233,9 @@ "outputs": [], "source": [ "optional_minus_graph = pynini.closure(\n", - " pynutil.insert(\"negative: \") + pynini.cross(\"moins\", \"\\\"-\\\"\") + \" \", 0, 1 # Note the extra space to separate the value from the integer field\n", + " pynutil.insert(\"negative: \") + pynini.cross(\"moins\", \"\\\"-\\\"\") + \" \",\n", + " 0,\n", + " 1, # Note the extra space to separate the value from the integer field\n", ")\n", "\n", "final_graph = optional_minus_graph + pynutil.insert(\"integer: \\\"\") + graph + pynutil.insert(\"\\\"\")" @@ -2214,10 +2262,10 @@ "class CardinalFst(GraphFst):\n", " def __init__(self):\n", " super().__init__(name=\"cardinal\", kind=\"classify\")\n", - " \n", + "\n", " ### Cardinal Grammar....\n", " ### .....\n", - " graph = graph_trillions | zero \n", + " graph = graph_trillions | zero\n", "\n", " ### Formatting grammar....\n", " ### .....\n", @@ -2225,12 +2273,12 @@ "\n", " ### Token insertion\n", " optional_minus_graph = pynini.closure(\n", - " pynutil.insert(\"negative: \") + pynini.cross(\"moins\", \"\\\"-\\\"\") + \" \", 0, 1\n", - " )\n", + " pynutil.insert(\"negative: \") + pynini.cross(\"moins\", \"\\\"-\\\"\") + \" \", 0, 1\n", + " )\n", "\n", " final_graph = optional_minus_graph + pynutil.insert(\"integer: \\\"\") + graph + pynutil.insert(\"\\\"\")\n", "\n", - " final_graph = self.add_tokens(final_graph) # inserts the cardinal tag\n", + " final_graph = self.add_tokens(final_graph) # inserts the cardinal tag\n", "\n", " self.fst = final_graph" ] @@ -2305,8 +2353,8 @@ "outputs": [], "source": [ "class CardinalFst(GraphFst):\n", - " def __init__(self):\n", - " super().__init__(name=\"cardinal\", kind=\"verbalize\")" + " def __init__(self):\n", + " super().__init__(name=\"cardinal\", kind=\"verbalize\")" ] }, { @@ -2345,11 +2393,11 @@ "outputs": [], "source": [ "class CardinalFst(GraphFst):\n", - " def __init__(self):\n", - " super().__init__(name=\"cardinal\", kind=\"verbalize\")\n", - " \n", - " # Removes the negative attribute and leaves the sign if occurs\n", - " optional_sign = pynini.closure(\n", + " def __init__(self):\n", + " super().__init__(name=\"cardinal\", kind=\"verbalize\")\n", + "\n", + " # Removes the negative attribute and leaves the sign if occurs\n", + " optional_sign = pynini.closure(\n", " pynutil.delete(\"negative:\")\n", " + delete_space\n", " + pynutil.delete(\"\\\"\")\n", @@ -2359,21 +2407,21 @@ " 0,\n", " 1,\n", " )\n", - " \n", - " # removes integer aspect\n", - " graph = (\n", + "\n", + " # removes integer aspect\n", + " graph = (\n", " pynutil.delete(\"integer:\")\n", " + delete_space\n", " + pynutil.delete(\"\\\"\")\n", - " + pynini.closure(NEMO_DIGIT, 1) # Accepts at least one digit\n", + " + pynini.closure(NEMO_DIGIT, 1) # Accepts at least one digit\n", " + pynutil.delete(\"\\\"\")\n", " )\n", - " \n", - " graph = optional_sign + graph # concatenates two properties\n", "\n", - " delete_tokens = self.delete_tokens(graph) # removes semiotic class tag\n", + " graph = optional_sign + graph # concatenates two properties\n", "\n", - " self.fst = delete_tokens.optimize()" + " delete_tokens = self.delete_tokens(graph) # removes semiotic class tag\n", + "\n", + " self.fst = delete_tokens.optimize()" ] }, { @@ -2519,8 +2567,8 @@ }, "outputs": [], "source": [ - "strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", - "graph_strip_morpheme = NEMO_SIGMA + strip_morpheme # accepts all strings until passed suffix, then deletes suffix" + "strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", + "graph_strip_morpheme = NEMO_SIGMA + strip_morpheme # accepts all strings until passed suffix, then deletes suffix" ] }, { @@ -2543,22 +2591,22 @@ "class CardinalFst(GraphFst):\n", " def __init__(self):\n", " super().__init__(name=\"cardinal\", kind=\"classify\")\n", - " \n", + "\n", " ### Cardinal Grammar....\n", " ### .....\n", - " graph = graph_trillions | zero \n", + " graph = graph_trillions | zero\n", "\n", " ### Formatting grammar....\n", " ### .....\n", " graph = graph @ clean_cardinal\n", - " \n", + "\n", " ### NEW GRAPH\n", - " self.just_cardinals = graph # will produce cardinals without formatting\n", + " self.just_cardinals = graph # will produce cardinals without formatting\n", "\n", " ### Token insertion\n", " optional_minus_graph = pynini.closure(\n", - " pynutil.insert(\"negative: \") + pynini.cross(\"moins\", \"\\\"-\\\"\") + \" \", 0, 1\n", - " )\n", + " pynutil.insert(\"negative: \") + pynini.cross(\"moins\", \"\\\"-\\\"\") + \" \", 0, 1\n", + " )\n", "\n", " final_graph = optional_minus_graph + pynutil.insert(\"integer: \\\"\") + graph + pynutil.insert(\"\\\"\")\n", "\n", @@ -2582,7 +2630,7 @@ }, "outputs": [], "source": [ - "graph_cardinal = CardinalFst().just_cardinals \n", + "graph_cardinal = CardinalFst().just_cardinals\n", "graph_ordinal_regular_suffix = graph_strip_morpheme @ graph_cardinal" ] }, @@ -2603,7 +2651,7 @@ }, "outputs": [], "source": [ - "example = \"sixième\" # dervied from six/6\n", + "example = \"sixième\" # dervied from six/6\n", "apply_fst(example, graph_ordinal_regular_suffix)" ] }, @@ -2632,21 +2680,24 @@ }, "outputs": [], "source": [ - "graph_root_change = pynini.string_map([(\"quatrième\", \"quatre\"),\n", - " (\"cinquième\",\t\"cinq\"),\n", - " (\"neuvième\",\t\"neuf\"),\n", - " (\"onzième\",\t\"onze\"),\n", - " (\"douzième\",\t\"douze\"),\n", - " (\"treizième\",\t\"treize\"),\n", - " (\"quatorzième\",\t\"quatorze\"),\n", - " (\"quinzième\",\t\"quinze\"),\n", - " (\"seizième\",\t\"seize\"),\n", - " (\"trentième\",\t\"trente\"),\n", - " (\"quarantième\",\t\"quarante\"),\n", - " (\"cinquantième\",\t\"cinquante\"),\n", - " (\"soixantième\",\t\"soixante\"),\n", - " (\"millième\",\t\"mille\"),\n", - "])" + "graph_root_change = pynini.string_map(\n", + " [\n", + " (\"quatrième\", \"quatre\"),\n", + " (\"cinquième\", \"cinq\"),\n", + " (\"neuvième\", \"neuf\"),\n", + " (\"onzième\", \"onze\"),\n", + " (\"douzième\", \"douze\"),\n", + " (\"treizième\", \"treize\"),\n", + " (\"quatorzième\", \"quatorze\"),\n", + " (\"quinzième\", \"quinze\"),\n", + " (\"seizième\", \"seize\"),\n", + " (\"trentième\", \"trente\"),\n", + " (\"quarantième\", \"quarante\"),\n", + " (\"cinquantième\", \"cinquante\"),\n", + " (\"soixantième\", \"soixante\"),\n", + " (\"millième\", \"mille\"),\n", + " ]\n", + ")" ] }, { @@ -2675,8 +2726,8 @@ }, "outputs": [], "source": [ - "graph_firsts = pynini.string_map([(\"premier\", \"un\"),(\"première\", \"un\")])\n", - "graph_seconds = pynini.string_map([(\"second\", \"deux\"),(\"seconde\", \"deux\")])" + "graph_firsts = pynini.string_map([(\"premier\", \"un\"), (\"première\", \"un\")])\n", + "graph_seconds = pynini.string_map([(\"second\", \"deux\"), (\"seconde\", \"deux\")])" ] }, { @@ -2707,31 +2758,34 @@ }, "outputs": [], "source": [ - "strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", - "\n", - "graph_root_change = pynini.string_map([(\"quatrième\", \"quatre\"),\n", - " (\"cinquième\",\t\"cinq\"),\n", - " (\"neuvième\",\t\"neuf\"),\n", - " (\"onzième\",\t\"onze\"),\n", - " (\"douzième\",\t\"douze\"),\n", - " (\"treizième\",\t\"treize\"),\n", - " (\"quatorzième\",\t\"quatorze\"),\n", - " (\"quinzième\",\t\"quinze\"),\n", - " (\"seizième\",\t\"seize\"),\n", - " (\"trentième\",\t\"trente\"),\n", - " (\"quarantième\",\t\"quarante\"),\n", - " (\"cinquantième\",\t\"cinquante\"),\n", - " (\"soixantième\",\t\"soixante\"),\n", - " (\"millième\",\t\"mille\"),\n", - "])\n", + "strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", + "\n", + "graph_root_change = pynini.string_map(\n", + " [\n", + " (\"quatrième\", \"quatre\"),\n", + " (\"cinquième\", \"cinq\"),\n", + " (\"neuvième\", \"neuf\"),\n", + " (\"onzième\", \"onze\"),\n", + " (\"douzième\", \"douze\"),\n", + " (\"treizième\", \"treize\"),\n", + " (\"quatorzième\", \"quatorze\"),\n", + " (\"quinzième\", \"quinze\"),\n", + " (\"seizième\", \"seize\"),\n", + " (\"trentième\", \"trente\"),\n", + " (\"quarantième\", \"quarante\"),\n", + " (\"cinquantième\", \"cinquante\"),\n", + " (\"soixantième\", \"soixante\"),\n", + " (\"millième\", \"mille\"),\n", + " ]\n", + ")\n", "\n", "# Component will accept all tokens that end with desired strings\n", - "graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change) \n", + "graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change)\n", "\n", - "graph_firsts = pynini.string_map([(\"premier\", \"un\"),(\"première\", \"un\")])\n", - "graph_seconds = pynini.string_map([(\"second\", \"deux\"),(\"seconde\", \"deux\")])\n", + "graph_firsts = pynini.string_map([(\"premier\", \"un\"), (\"première\", \"un\")])\n", + "graph_seconds = pynini.string_map([(\"second\", \"deux\"), (\"seconde\", \"deux\")])\n", "\n", - "graph_get_cardinal = pynini.union(graph_firsts, graph_seconds, graph_get_cardinal) \n", + "graph_get_cardinal = pynini.union(graph_firsts, graph_seconds, graph_get_cardinal)\n", "\n", "graph_cardinal = CardinalFst().just_cardinals\n", "\n", @@ -2794,7 +2848,7 @@ "outputs": [], "source": [ "def __init__(self, cardinal: GraphFst):\n", - " super().__init__(name=\"ordinal\", kind=\"classify\")" + " super().__init__(name=\"ordinal\", kind=\"classify\")" ] }, { @@ -2816,10 +2870,11 @@ "source": [ "from nemo_text_processing.inverse_text_normalization.fr.taggers.cardinal import CardinalFst\n", "\n", + "\n", "class OrdinalFst(GraphFst):\n", - " def __init__(self, cardinal: GraphFst):\n", - " super().__init__(name=\"ordinal\", kind=\"classify\")\n", - " graph_cardinal = cardinal.graph_no_exception # NeMo equivalent to self.just_cardinals" + " def __init__(self, cardinal: GraphFst):\n", + " super().__init__(name=\"ordinal\", kind=\"classify\")\n", + " graph_cardinal = cardinal.graph_no_exception # NeMo equivalent to self.just_cardinals" ] }, { @@ -2840,37 +2895,40 @@ "outputs": [], "source": [ "class OrdinalFst(GraphFst):\n", - " def __init__(self, cardinal: GraphFst):\n", - " super().__init__(name=\"ordinal\", kind=\"classify\")\n", - " graph_cardinal = cardinal.graph_no_exception # may replace\n", - "\n", - " strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", + " def __init__(self, cardinal: GraphFst):\n", + " super().__init__(name=\"ordinal\", kind=\"classify\")\n", + " graph_cardinal = cardinal.graph_no_exception # may replace\n", + "\n", + " strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", + "\n", + " graph_root_change = pynini.string_map(\n", + " [\n", + " (\"quatrième\", \"quatre\"),\n", + " (\"cinquième\", \"cinq\"),\n", + " (\"neuvième\", \"neuf\"),\n", + " (\"onzième\", \"onze\"),\n", + " (\"douzième\", \"douze\"),\n", + " (\"treizième\", \"treize\"),\n", + " (\"quatorzième\", \"quatorze\"),\n", + " (\"quinzième\", \"quinze\"),\n", + " (\"seizième\", \"seize\"),\n", + " (\"trentième\", \"trente\"),\n", + " (\"quarantième\", \"quarante\"),\n", + " (\"cinquantième\", \"cinquante\"),\n", + " (\"soixantième\", \"soixante\"),\n", + " (\"millième\", \"mille\"),\n", + " ]\n", + " )\n", "\n", - " graph_root_change = pynini.string_map([(\"quatrième\", \"quatre\"),\n", - " (\"cinquième\",\t\"cinq\"),\n", - " (\"neuvième\",\t\"neuf\"),\n", - " (\"onzième\",\t\"onze\"),\n", - " (\"douzième\",\t\"douze\"),\n", - " (\"treizième\",\t\"treize\"),\n", - " (\"quatorzième\",\t\"quatorze\"),\n", - " (\"quinzième\",\t\"quinze\"),\n", - " (\"seizième\",\t\"seize\"),\n", - " (\"trentième\",\t\"trente\"),\n", - " (\"quarantième\",\t\"quarante\"),\n", - " (\"cinquantième\",\t\"cinquante\"),\n", - " (\"soixantième\",\t\"soixante\"),\n", - " (\"millième\",\t\"mille\"),\n", - " ])\n", - " \n", - " # Component will accept all tokens that end with desired strings\n", - " graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change) \n", + " # Component will accept all tokens that end with desired strings\n", + " graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change)\n", "\n", - " graph_firsts = pynini.string_map([(\"premier\", \"un\"),(\"première\", \"un\")])\n", - " graph_seconds = pynini.string_map([(\"second\", \"deux\"),(\"seconde\", \"deux\")])\n", + " graph_firsts = pynini.string_map([(\"premier\", \"un\"), (\"première\", \"un\")])\n", + " graph_seconds = pynini.string_map([(\"second\", \"deux\"), (\"seconde\", \"deux\")])\n", "\n", - " graph_get_cardinal = pynini.union(graph_firsts, graph_seconds, graph_get_cardinal) \n", + " graph_get_cardinal = pynini.union(graph_firsts, graph_seconds, graph_get_cardinal)\n", "\n", - " graph_ordinal = graph_get_cardinal @ graph_cardinal\n" + " graph_ordinal = graph_get_cardinal @ graph_cardinal" ] }, { @@ -2906,14 +2964,12 @@ }, "outputs": [], "source": [ - "graph_morpheme = pynutil.insert(\"e\") # Insert e superscript\n", - "graph_plural = pynini.closure(pynini.accep(\"s\"), 0, 1) # We create an acceptor since we must process the possible \"s\"\n", + "graph_morpheme = pynutil.insert(\"e\") # Insert e superscript\n", + "graph_plural = pynini.closure(pynini.accep(\"s\"), 0, 1) # We create an acceptor since we must process the possible \"s\"\n", "\n", "graph_morpheme_component = graph_morpheme + graph_plural\n", "\n", - "graph_morphosyntactic_features = (pynutil.insert(\" morphosyntactic_features: \\\"\") \n", - " + graph_morpheme_component\n", - " )" + "graph_morphosyntactic_features = pynutil.insert(\" morphosyntactic_features: \\\"\") + graph_morpheme_component" ] }, { @@ -2933,7 +2989,7 @@ }, "outputs": [], "source": [ - "graph_reg_ordinals = graph_get_cardinal @ graph_cardinal # Rewriting ordinals to remove the first and second ordinal.\n", + "graph_reg_ordinals = graph_get_cardinal @ graph_cardinal # Rewriting ordinals to remove the first and second ordinal.\n", "\n", "graph_ordinal = pynutil.insert(\"integer: \\\"\") + graph_reg_ordinals + pynutil.insert(\"\\\"\")\n", "graph_ordinal += graph_morphosyntactic_features" @@ -2956,10 +3012,10 @@ }, "outputs": [], "source": [ - "firsts = pynini.string_map([(\"premier\", \"er\"), (\"première\",\"re\")])\n", - "firsts += graph_plural # Still accepts plural marker in superscript\n", - "seconds = pynini.string_map([(\"second\", \"d\"),(\"seconde\", \"de\")])\n", - "seconds += graph_plural \n", + "firsts = pynini.string_map([(\"premier\", \"er\"), (\"première\", \"re\")])\n", + "firsts += graph_plural # Still accepts plural marker in superscript\n", + "seconds = pynini.string_map([(\"second\", \"d\"), (\"seconde\", \"de\")])\n", + "seconds += graph_plural\n", "\n", "graph_firsts = pynutil.insert(\"integer: \\\"1\\\" morphosyntactic_features: \\\"\") + firsts\n", "graph_seconds = pynutil.insert(\"integer: \\\"2\\\" morphosyntactic_features: \\\"\") + seconds" @@ -2983,60 +3039,61 @@ "outputs": [], "source": [ "class OrdinalFst(GraphFst):\n", - " def __init__(self, cardinal: GraphFst):\n", - " super().__init__(name=\"ordinal\", kind=\"classify\")\n", - " graph_cardinal = cardinal.graph_no_exception # may replace\n", - "\n", - " strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", - "\n", - " graph_root_change = pynini.string_map([(\"quatrième\", \"quatre\"),\n", - " (\"cinquième\",\t\"cinq\"),\n", - " (\"neuvième\",\t\"neuf\"),\n", - " (\"onzième\",\t\"onze\"),\n", - " (\"douzième\",\t\"douze\"),\n", - " (\"treizième\",\t\"treize\"),\n", - " (\"quatorzième\",\t\"quatorze\"),\n", - " (\"quinzième\",\t\"quinze\"),\n", - " (\"seizième\",\t\"seize\"),\n", - " (\"trentième\",\t\"trente\"),\n", - " (\"quarantième\",\t\"quarante\"),\n", - " (\"cinquantième\",\t\"cinquante\"),\n", - " (\"soixantième\",\t\"soixante\"),\n", - " (\"millième\",\t\"mille\"),\n", - " ])\n", - " \n", - " # Component will accept all tokens that end with desired strings\n", - " graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change) \n", - "\n", - " # Graph will map ordinals beyond second ordinal to their cardinals\n", - " graph_reg_ordinals = graph_get_cardinal @ graph_cardinal\n", - "\n", - " # Graphing morphosyntactic_features\n", - " graph_morpheme = pynutil.insert(\"e\") # Insert e superscript\n", - " graph_plural = pynini.accep(\"s\").ques # ques is equivalent to pynini.closure(, 0, 1)\n", - "\n", - " graph_morpheme_component = graph_morpheme + graph_plural\n", - "\n", - " graph_morphosyntactic_features = (pynutil.insert(\" morphosyntactic_features: \\\"\") \n", - " + graph_morpheme_component\n", + " def __init__(self, cardinal: GraphFst):\n", + " super().__init__(name=\"ordinal\", kind=\"classify\")\n", + " graph_cardinal = cardinal.graph_no_exception # may replace\n", + "\n", + " strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", + "\n", + " graph_root_change = pynini.string_map(\n", + " [\n", + " (\"quatrième\", \"quatre\"),\n", + " (\"cinquième\", \"cinq\"),\n", + " (\"neuvième\", \"neuf\"),\n", + " (\"onzième\", \"onze\"),\n", + " (\"douzième\", \"douze\"),\n", + " (\"treizième\", \"treize\"),\n", + " (\"quatorzième\", \"quatorze\"),\n", + " (\"quinzième\", \"quinze\"),\n", + " (\"seizième\", \"seize\"),\n", + " (\"trentième\", \"trente\"),\n", + " (\"quarantième\", \"quarante\"),\n", + " (\"cinquantième\", \"cinquante\"),\n", + " (\"soixantième\", \"soixante\"),\n", + " (\"millième\", \"mille\"),\n", + " ]\n", " )\n", "\n", - " # Adding in the `integer` property:\n", - " graph_ordinal = pynutil.insert(\"integer: \\\"\") + graph_reg_ordinals + pynutil.insert(\"\\\"\")\n", - " graph_ordinal += graph_morphosyntactic_features \n", + " # Component will accept all tokens that end with desired strings\n", + " graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change)\n", + "\n", + " # Graph will map ordinals beyond second ordinal to their cardinals\n", + " graph_reg_ordinals = graph_get_cardinal @ graph_cardinal\n", + "\n", + " # Graphing morphosyntactic_features\n", + " graph_morpheme = pynutil.insert(\"e\") # Insert e superscript\n", + " graph_plural = pynini.accep(\"s\").ques # ques is equivalent to pynini.closure(, 0, 1)\n", "\n", - " # Case of first and second ordinals\n", - " firsts = pynini.string_map([(\"premier\", \"er\"), (\"première\",\"re\")])\n", - " firsts += graph_plural # Still accepts plural marker in superscript\n", - " seconds = pynini.string_map([(\"second\", \"d\"),(\"seconde\", \"de\")])\n", - " seconds += graph_plural \n", + " graph_morpheme_component = graph_morpheme + graph_plural\n", "\n", - " graph_firsts = pynutil.insert(\"integer: \\\"1\\\" morphosyntactic_features: \\\"\") + firsts\n", - " graph_seconds = pynutil.insert(\"integer: \\\"2\\\" morphosyntactic_features: \\\"\") + seconds\n", + " graph_morphosyntactic_features = pynutil.insert(\" morphosyntactic_features: \\\"\") + graph_morpheme_component\n", "\n", - " # All together\n", - " graph_ordinal = pynini.union(graph_ordinal, graph_firsts, graph_seconds)\n", - " self.fst = graph_ordinal.optimize()" + " # Adding in the `integer` property:\n", + " graph_ordinal = pynutil.insert(\"integer: \\\"\") + graph_reg_ordinals + pynutil.insert(\"\\\"\")\n", + " graph_ordinal += graph_morphosyntactic_features\n", + "\n", + " # Case of first and second ordinals\n", + " firsts = pynini.string_map([(\"premier\", \"er\"), (\"première\", \"re\")])\n", + " firsts += graph_plural # Still accepts plural marker in superscript\n", + " seconds = pynini.string_map([(\"second\", \"d\"), (\"seconde\", \"de\")])\n", + " seconds += graph_plural\n", + "\n", + " graph_firsts = pynutil.insert(\"integer: \\\"1\\\" morphosyntactic_features: \\\"\") + firsts\n", + " graph_seconds = pynutil.insert(\"integer: \\\"2\\\" morphosyntactic_features: \\\"\") + seconds\n", + "\n", + " # All together\n", + " graph_ordinal = pynini.union(graph_ordinal, graph_firsts, graph_seconds)\n", + " self.fst = graph_ordinal.optimize()" ] }, { @@ -3102,7 +3159,9 @@ "source": [ "special_tokens = pynini.accep(\"siècle\")\n", "\n", - "graph_special_tokens = delete_space + pynutil.insert(\"/\") + special_tokens # We need to delete the space in between this token and the following one.\n", + "graph_special_tokens = (\n", + " delete_space + pynutil.insert(\"/\") + special_tokens\n", + ") # We need to delete the space in between this token and the following one.\n", "graph_special_tokens = pynini.closure(graph_special_tokens, 0, 1)\n", "\n", "graph_ordinal += graph_special_tokens + pynutil.insert(\"\\\"\")" @@ -3128,69 +3187,71 @@ "outputs": [], "source": [ "class OrdinalFst(GraphFst):\n", - " def __init__(self, cardinal: GraphFst):\n", - " super().__init__(name=\"ordinal\", kind=\"classify\")\n", - " graph_cardinal = cardinal.graph_no_exception # may replace\n", - "\n", - " strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", - "\n", - " graph_root_change = pynini.string_map([(\"quatrième\", \"quatre\"),\n", - " (\"cinquième\",\t\"cinq\"),\n", - " (\"neuvième\",\t\"neuf\"),\n", - " (\"onzième\",\t\"onze\"),\n", - " (\"douzième\",\t\"douze\"),\n", - " (\"treizième\",\t\"treize\"),\n", - " (\"quatorzième\",\t\"quatorze\"),\n", - " (\"quinzième\",\t\"quinze\"),\n", - " (\"seizième\",\t\"seize\"),\n", - " (\"trentième\",\t\"trente\"),\n", - " (\"quarantième\",\t\"quarante\"),\n", - " (\"cinquantième\",\t\"cinquante\"),\n", - " (\"soixantième\",\t\"soixante\"),\n", - " (\"millième\",\t\"mille\"),\n", - " ])\n", - " \n", - " # Component will accept all tokens that end with desired strings\n", - " graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change) \n", - "\n", - " # Graph will map ordinals beyond second ordinal to their cardinals\n", - " graph_reg_ordinals = graph_get_cardinal @ graph_cardinal\n", - "\n", - " # Graphing morphosyntactic_features\n", - " graph_morpheme = pynutil.insert(\"e\") # Insert e superscript\n", - " graph_plural = pynini.accep(\"s\").ques # We create an acceptor since we must process the possible \"s\"\n", - "\n", - " graph_morpheme_component = graph_morpheme + graph_plural\n", - "\n", - " graph_morphosyntactic_features = (pynutil.insert(\" morphosyntactic_features: \\\"\") \n", - " + graph_morpheme_component\n", + " def __init__(self, cardinal: GraphFst):\n", + " super().__init__(name=\"ordinal\", kind=\"classify\")\n", + " graph_cardinal = cardinal.graph_no_exception # may replace\n", + "\n", + " strip_morpheme = pynutil.delete(\"ième\") # deletes suffix\n", + "\n", + " graph_root_change = pynini.string_map(\n", + " [\n", + " (\"quatrième\", \"quatre\"),\n", + " (\"cinquième\", \"cinq\"),\n", + " (\"neuvième\", \"neuf\"),\n", + " (\"onzième\", \"onze\"),\n", + " (\"douzième\", \"douze\"),\n", + " (\"treizième\", \"treize\"),\n", + " (\"quatorzième\", \"quatorze\"),\n", + " (\"quinzième\", \"quinze\"),\n", + " (\"seizième\", \"seize\"),\n", + " (\"trentième\", \"trente\"),\n", + " (\"quarantième\", \"quarante\"),\n", + " (\"cinquantième\", \"cinquante\"),\n", + " (\"soixantième\", \"soixante\"),\n", + " (\"millième\", \"mille\"),\n", + " ]\n", " )\n", "\n", - " # Adding in the `integer` property:\n", - " graph_ordinal = pynutil.insert(\"integer: \\\"\") + graph_reg_ordinals + pynutil.insert(\"\\\"\")\n", - " graph_ordinal += graph_morphosyntactic_features \n", + " # Component will accept all tokens that end with desired strings\n", + " graph_get_cardinal = NEMO_SIGMA + (strip_morpheme | graph_root_change)\n", + "\n", + " # Graph will map ordinals beyond second ordinal to their cardinals\n", + " graph_reg_ordinals = graph_get_cardinal @ graph_cardinal\n", + "\n", + " # Graphing morphosyntactic_features\n", + " graph_morpheme = pynutil.insert(\"e\") # Insert e superscript\n", + " graph_plural = pynini.accep(\"s\").ques # We create an acceptor since we must process the possible \"s\"\n", "\n", - " # Case of first and second ordinals\n", - " firsts = pynini.string_map([(\"premier\", \"er\"), (\"première\",\"re\")])\n", - " firsts += graph_plural # Still accepts plural marker in superscript\n", - " seconds = pynini.string_map([(\"second\", \"d\"),(\"seconde\", \"de\")])\n", - " seconds += graph_plural \n", + " graph_morpheme_component = graph_morpheme + graph_plural\n", "\n", - " graph_firsts = pynutil.insert(\"integer: \\\"1\\\" morphosyntactic_features: \\\"\") + firsts\n", - " graph_seconds = pynutil.insert(\"integer: \\\"2\\\" morphosyntactic_features: \\\"\") + seconds\n", + " graph_morphosyntactic_features = pynutil.insert(\" morphosyntactic_features: \\\"\") + graph_morpheme_component\n", "\n", + " # Adding in the `integer` property:\n", + " graph_ordinal = pynutil.insert(\"integer: \\\"\") + graph_reg_ordinals + pynutil.insert(\"\\\"\")\n", + " graph_ordinal += graph_morphosyntactic_features\n", "\n", - " # Special tokens\n", - " special_tokens = pynini.accep(\"siècle\")\n", + " # Case of first and second ordinals\n", + " firsts = pynini.string_map([(\"premier\", \"er\"), (\"première\", \"re\")])\n", + " firsts += graph_plural # Still accepts plural marker in superscript\n", + " seconds = pynini.string_map([(\"second\", \"d\"), (\"seconde\", \"de\")])\n", + " seconds += graph_plural\n", "\n", - " graph_special_tokens = delete_space + pynutil.insert(\"/\") + special_tokens # We need to delete the space in between this token and the following one.\n", - " graph_special_tokens = pynini.closure(graph_special_tokens, 0, 1)\n", + " graph_firsts = pynutil.insert(\"integer: \\\"1\\\" morphosyntactic_features: \\\"\") + firsts\n", + " graph_seconds = pynutil.insert(\"integer: \\\"2\\\" morphosyntactic_features: \\\"\") + seconds\n", "\n", - " graph_ordinal += graph_special_tokens + pynutil.insert(\"\\\"\")\n", + " # Special tokens\n", + " special_tokens = pynini.accep(\"siècle\")\n", "\n", - " # Finishing\n", - " graph_ordinal = self.add_tokens(graph_ordinal)\n", - " self.fst = graph_ordinal.optimize()\n" + " graph_special_tokens = (\n", + " delete_space + pynutil.insert(\"/\") + special_tokens\n", + " ) # We need to delete the space in between this token and the following one.\n", + " graph_special_tokens = pynini.closure(graph_special_tokens, 0, 1)\n", + "\n", + " graph_ordinal += graph_special_tokens + pynutil.insert(\"\\\"\")\n", + "\n", + " # Finishing\n", + " graph_ordinal = self.add_tokens(graph_ordinal)\n", + " self.fst = graph_ordinal.optimize()" ] }, { @@ -3248,16 +3309,16 @@ }, "outputs": [], "source": [ - " # Create mappings for all superscripts\n", - " superscript = pynini.union(\n", - " pynini.cross(\"e\", \"ᵉ\"), # only delete first quote since there may be more features\n", - " pynini.cross(\"d\", \"ᵈ\"),\n", - " pynini.cross(\"r\", \"ʳ\"),\n", - " pynini.cross(\"s\", \"ˢ\"),\n", - " )\n", + "# Create mappings for all superscripts\n", + "superscript = pynini.union(\n", + " pynini.cross(\"e\", \"ᵉ\"), # only delete first quote since there may be more features\n", + " pynini.cross(\"d\", \"ᵈ\"),\n", + " pynini.cross(\"r\", \"ʳ\"),\n", + " pynini.cross(\"s\", \"ˢ\"),\n", + ")\n", "\n", - " # Append to deletion of feature property. Note that we use plus closure for multiple superscripts.\n", - " graph_morphosyntactic_features = pynutil.delete(\" morphosyntactic_features: \\\"\") + superscript.plus" + "# Append to deletion of feature property. Note that we use plus closure for multiple superscripts.\n", + "graph_morphosyntactic_features = pynutil.delete(\" morphosyntactic_features: \\\"\") + superscript.plus" ] }, { @@ -3307,12 +3368,12 @@ "outputs": [], "source": [ "graph_integer = (\n", - " pynutil.delete(\"integer:\")\n", - " + delete_space\n", - " + pynutil.delete(\"\\\"\")\n", - " + pynini.closure(NEMO_DIGIT, 1)\n", - " + pynutil.delete(\"\\\"\")\n", - " )\n", + " pynutil.delete(\"integer:\")\n", + " + delete_space\n", + " + pynutil.delete(\"\\\"\")\n", + " + pynini.closure(NEMO_DIGIT, 1)\n", + " + pynutil.delete(\"\\\"\")\n", + ")\n", "graph_Arabic = graph_integer + graph_morphosyntactic_features + pynutil.delete(\"\\\"\")" ] }, @@ -3351,27 +3412,33 @@ }, "outputs": [], "source": [ - "digits = pynini.string_map([(\"1\", \"I\"),\n", - " (\"2\",\t\"II\"),\n", - " (\"3\",\t\"III\"),\n", - " (\"4\",\t\"IV\"),\n", - " (\"5\",\t\"V\"),\n", - " (\"6\",\t\"VI\"),\n", - " (\"7\",\t\"VII\"),\n", - " (\"8\",\t\"VIII\"),\n", - " (\"9\",\t\"IX\"),\n", - " ])\n", - "tens = pynini.string_map([(\"1\", \"X\"),\n", - " (\"2\",\t\"XX\"),\n", - " (\"3\",\t\"XXX\"),\n", - " (\"4\",\t\"XL\"),\n", - " (\"5\",\t\"L\"),\n", - " (\"6\",\t\"LX\"),\n", - " (\"7\",\t\"LXX\"),\n", - " (\"8\",\t\"LXXX\"),\n", - " (\"9\",\t\"XC\"),\n", - " ])\n", - "zero = pynutil.delete(\"0\") # No Roman representation for zero." + "digits = pynini.string_map(\n", + " [\n", + " (\"1\", \"I\"),\n", + " (\"2\", \"II\"),\n", + " (\"3\", \"III\"),\n", + " (\"4\", \"IV\"),\n", + " (\"5\", \"V\"),\n", + " (\"6\", \"VI\"),\n", + " (\"7\", \"VII\"),\n", + " (\"8\", \"VIII\"),\n", + " (\"9\", \"IX\"),\n", + " ]\n", + ")\n", + "tens = pynini.string_map(\n", + " [\n", + " (\"1\", \"X\"),\n", + " (\"2\", \"XX\"),\n", + " (\"3\", \"XXX\"),\n", + " (\"4\", \"XL\"),\n", + " (\"5\", \"L\"),\n", + " (\"6\", \"LX\"),\n", + " (\"7\", \"LXX\"),\n", + " (\"8\", \"LXXX\"),\n", + " (\"9\", \"XC\"),\n", + " ]\n", + ")\n", + "zero = pynutil.delete(\"0\") # No Roman representation for zero." ] }, { @@ -3392,7 +3459,7 @@ "outputs": [], "source": [ "map_one_digit = NEMO_DIGIT\n", - "map_two_digits = NEMO_DIGIT ** 2 # pynini overloads the exponent function to allow self-concatenation." + "map_two_digits = NEMO_DIGIT**2 # pynini overloads the exponent function to allow self-concatenation." ] }, { @@ -3486,31 +3553,37 @@ " graph_Arabic = graph_integer + graph_morphosyntactic_features + pynutil.delete(\"\\\"\")\n", "\n", " # Mapping Roman numerals\n", - " digits = pynini.string_map([(\"1\", \"I\"),\n", - " (\"2\",\t\"II\"),\n", - " (\"3\",\t\"III\"),\n", - " (\"4\",\t\"IV\"),\n", - " (\"5\",\t\"V\"),\n", - " (\"6\",\t\"VI\"),\n", - " (\"7\",\t\"VII\"),\n", - " (\"8\",\t\"VIII\"),\n", - " (\"9\",\t\"IX\"),\n", - " ])\n", - " tens = pynini.string_map([(\"1\", \"X\"),\n", - " (\"2\",\t\"XX\"),\n", - " (\"3\",\t\"XXX\"),\n", - " (\"4\",\t\"XL\"),\n", - " (\"5\",\t\"L\"),\n", - " (\"6\",\t\"LX\"),\n", - " (\"7\",\t\"LXX\"),\n", - " (\"8\",\t\"LXXX\"),\n", - " (\"9\",\t\"XC\"),\n", - " ])\n", - " zero = pynutil.delete(\"0\") # No Roman representation for zero.\n", + " digits = pynini.string_map(\n", + " [\n", + " (\"1\", \"I\"),\n", + " (\"2\", \"II\"),\n", + " (\"3\", \"III\"),\n", + " (\"4\", \"IV\"),\n", + " (\"5\", \"V\"),\n", + " (\"6\", \"VI\"),\n", + " (\"7\", \"VII\"),\n", + " (\"8\", \"VIII\"),\n", + " (\"9\", \"IX\"),\n", + " ]\n", + " )\n", + " tens = pynini.string_map(\n", + " [\n", + " (\"1\", \"X\"),\n", + " (\"2\", \"XX\"),\n", + " (\"3\", \"XXX\"),\n", + " (\"4\", \"XL\"),\n", + " (\"5\", \"L\"),\n", + " (\"6\", \"LX\"),\n", + " (\"7\", \"LXX\"),\n", + " (\"8\", \"LXXX\"),\n", + " (\"9\", \"XC\"),\n", + " ]\n", + " )\n", + " zero = pynutil.delete(\"0\") # No Roman representation for zero.\n", "\n", " # filters for Roman digits\n", " map_one_digit = NEMO_DIGIT\n", - " map_two_digits = NEMO_DIGIT ** 2 # pynini overloads the exponent function to allow self-concatenation.\n", + " map_two_digits = NEMO_DIGIT**2 # pynini overloads the exponent function to allow self-concatenation.\n", "\n", " # Composing onto roman digits\n", " graph_one_digit_romans = NEMO_DIGIT @ digits\n", @@ -3525,10 +3598,10 @@ " graph_romans += pynini.cross(\"/\", \" \") + \"siècle\" + pynutil.delete(\"\\\"\")\n", "\n", " # Final composition\n", - " graph = (graph_romans | graph_Arabic)\n", + " graph = graph_romans | graph_Arabic\n", "\n", " delete_tokens = self.delete_tokens(graph)\n", - " self.fst = delete_tokens.optimize()\n" + " self.fst = delete_tokens.optimize()" ] }, { @@ -3643,7 +3716,7 @@ }, "outputs": [], "source": [ - "cardinal = CardinalFst().graph_no_exception # NeMo equivalent of just_cardinals\n", + "cardinal = CardinalFst().graph_no_exception # NeMo equivalent of just_cardinals\n", "\n", "# place cardinal under closure to permit values <=1\n", "graph_integer = pynini.closure(cardinal, 0, 1)" @@ -3710,8 +3783,8 @@ }, "outputs": [], "source": [ - "example = \"trois virgule trois cinquante-cinq\" \n", - "apply_fst(example, graph_decimal) # Should output only the cardinals in the string" + "example = \"trois virgule trois cinquante-cinq\"\n", + "apply_fst(example, graph_decimal) # Should output only the cardinals in the string" ] }, { @@ -3813,7 +3886,9 @@ }, "outputs": [], "source": [ - "graph_integer_or_none = graph_integer | pynutil.insert(\"integer_part: \\\"0\\\" \", weight=.1) # In cases we don't always have an integer preceding\n", + "graph_integer_or_none = graph_integer | pynutil.insert(\n", + " \"integer_part: \\\"0\\\" \", weight=0.1\n", + ") # In cases we don't always have an integer preceding\n", "graph_decimal_no_sign = graph_integer_or_none + delete_space + pynutil.delete(\"virgule\") + graph_fractional" ] }, @@ -3876,13 +3951,15 @@ " delete_virgule = pynutil.delete(\"virgule\")\n", "\n", " graph_integer = pynutil.insert(\"integer_part: \\\"\") + cardinal + pynutil.insert(\"\\\" \") + delete_space\n", - " graph_integer_or_none = graph_integer | pynutil.insert(\"integer_part: \\\"0\\\" \", weight=.001) # In cases we don't always have an integer preceding\n", + " graph_integer_or_none = graph_integer | pynutil.insert(\n", + " \"integer_part: \\\"0\\\" \", weight=0.001\n", + " ) # In cases we don't always have an integer preceding\n", "\n", " graph_string_of_cardinals = delete_space + cardinal\n", " graph_string_of_cardinals = pynini.closure(graph_string_of_cardinals, 1)\n", " graph_fractional = pynutil.insert(\"fractional_part: \\\"\") + graph_string_of_cardinals + pynutil.insert(\"\\\"\")\n", "\n", - " graph_decimal_no_sign = graph_integer_or_none + pynutil.delete(\"virgule\") + graph_fractional \n", + " graph_decimal_no_sign = graph_integer_or_none + pynutil.delete(\"virgule\") + graph_fractional\n", "\n", " graph_negative = pynini.cross(\"moins\", \"negative: \\\"-\\\" \") + delete_space\n", " graph_negative = pynini.closure(graph_negative, 0, 1)\n", @@ -3890,8 +3967,7 @@ " graph_decimal = graph_negative + graph_decimal_no_sign\n", "\n", " graph = self.add_tokens(graph_decimal)\n", - " self.fst = graph.optimize()\n", - "\n" + " self.fst = graph.optimize()" ] }, { @@ -3936,19 +4012,19 @@ "outputs": [], "source": [ "suffix = pynini.union(\n", - " \"million\",\n", - " \"millions\",\n", - " \"milliard\",\n", - " \"milliards\",\n", - " \"billion\",\n", - " \"billions\",\n", - " \"billiard\",\n", - " \"billiards\",\n", - " \"trillion\",\n", - " \"trillions\",\n", - " \"trilliard\",\n", - " \"trilliards\",\n", - " )" + " \"million\",\n", + " \"millions\",\n", + " \"milliard\",\n", + " \"milliards\",\n", + " \"billion\",\n", + " \"billions\",\n", + " \"billiard\",\n", + " \"billiards\",\n", + " \"trillion\",\n", + " \"trillions\",\n", + " \"trilliard\",\n", + " \"trilliards\",\n", + ")" ] }, { @@ -3971,7 +4047,7 @@ "outputs": [], "source": [ "def get_quantity(decimal, cardinal_up_to_thousand):\n", - " key_values = pynini.union(\n", + " key_values = pynini.union(\n", " \"million\",\n", " \"millions\",\n", " \"milliard\",\n", @@ -3986,23 +4062,23 @@ " \"trilliards\",\n", " )\n", " # The French WFST that this borrows from has not removed leading zeroes yet.\n", - " numbers = cardinal_up_to_thousand @ (\n", - " pynutil.delete(pynini.closure(\"0\")) + pynini.difference(NEMO_DIGIT, \"0\") + pynini.closure(NEMO_DIGIT)\n", - " )\n", - " res = (\n", - " pynutil.insert(\"integer_part: \\\"\")\n", - " + numbers\n", - " + pynutil.insert(\"\\\"\")\n", - " + (\n", - " pynini.union(delete_hyphen, delete_extra_space)\n", - " ) # Can be written either as 'deux-millions' or 'deux millions' depending on whether it registers as a noun or part of cardinal.\n", - " + pynutil.insert(\" quantity: \\\"\")\n", - " + suffix\n", - " + pynutil.insert(\"\\\"\")\n", - " )\n", - " # Union with decimal to permit either a cardinal or decimal representation.\n", - " res |= decimal + delete_extra_space + pynutil.insert(\" quantity: \\\"\") + suffix + pynutil.insert(\"\\\"\")\n", - " return res" + " numbers = cardinal_up_to_thousand @ (\n", + " pynutil.delete(pynini.closure(\"0\")) + pynini.difference(NEMO_DIGIT, \"0\") + pynini.closure(NEMO_DIGIT)\n", + " )\n", + " res = (\n", + " pynutil.insert(\"integer_part: \\\"\")\n", + " + numbers\n", + " + pynutil.insert(\"\\\"\")\n", + " + (\n", + " pynini.union(delete_hyphen, delete_extra_space)\n", + " ) # Can be written either as 'deux-millions' or 'deux millions' depending on whether it registers as a noun or part of cardinal.\n", + " + pynutil.insert(\" quantity: \\\"\")\n", + " + suffix\n", + " + pynutil.insert(\"\\\"\")\n", + " )\n", + " # Union with decimal to permit either a cardinal or decimal representation.\n", + " res |= decimal + delete_extra_space + pynutil.insert(\" quantity: \\\"\") + suffix + pynutil.insert(\"\\\"\")\n", + " return res" ] }, { @@ -4030,22 +4106,22 @@ " delete_virgule = pynutil.delete(\"virgule\")\n", "\n", " graph_integer = pynutil.insert(\"integer_part: \\\"\") + cardinal + pynutil.insert(\"\\\" \") + delete_space\n", - " graph_integer_or_none = graph_integer | pynutil.insert(\"integer_part: \\\"0\\\" \", weight=.001) # In cases we don't always have an integer preceding\n", + " graph_integer_or_none = graph_integer | pynutil.insert(\n", + " \"integer_part: \\\"0\\\" \", weight=0.001\n", + " ) # In cases we don't always have an integer preceding\n", "\n", " graph_string_of_cardinals = delete_space + cardinal\n", " graph_string_of_cardinals = pynini.closure(graph_string_of_cardinals, 1)\n", " graph_fractional = pynutil.insert(\"fractional_part: \\\"\") + graph_string_of_cardinals + pynutil.insert(\"\\\"\")\n", "\n", - " graph_decimal_no_sign = graph_integer_or_none + delete_virgule + graph_fractional \n", + " graph_decimal_no_sign = graph_integer_or_none + delete_virgule + graph_fractional\n", "\n", " graph_negative = pynini.cross(\"moins\", \"negative: \\\"-\\\" \") + delete_space\n", " graph_negative = pynini.closure(graph_negative, 0, 1)\n", " graph_decimal = graph_negative + graph_decimal_no_sign\n", "\n", " # Union default decimal with version that accepts quantities\n", - " graph_decimal |= graph_negative + get_quantity(\n", - " graph_decimal_no_sign, quantities_cardinal\n", - " )\n", + " graph_decimal |= graph_negative + get_quantity(graph_decimal_no_sign, quantities_cardinal)\n", " final_graph = self.add_tokens(graph_decimal)\n", " self.fst = final_graph.optimize()" ] @@ -4114,8 +4190,10 @@ }, "outputs": [], "source": [ - "every_three_digits = NEMO_DIGIT ** 3 # accepts a string of three digits\n", - "space_every_three_integer = pynini.closure(NEMO_NON_BREAKING_SPACE + every_three_digits) # inserts space before every three digits." + "every_three_digits = NEMO_DIGIT**3 # accepts a string of three digits\n", + "space_every_three_integer = pynini.closure(\n", + " NEMO_NON_BREAKING_SPACE + every_three_digits\n", + ") # inserts space before every three digits." ] }, { @@ -4186,7 +4264,7 @@ " super().__init__(name=\"decimal\", kind=\"verbalize\")\n", "\n", " # Need parser to group digits by threes\n", - " exactly_three_digits = NEMO_DIGIT ** 3\n", + " exactly_three_digits = NEMO_DIGIT**3\n", " at_most_three_digits = pynini.closure(NEMO_DIGIT, 1, 3)\n", "\n", " space_every_three_integer = (\n", @@ -4229,7 +4307,7 @@ " )\n", " optional_quantity = pynini.closure(pynutil.insert(\" \") + quantity + delete_space, 0, 1)\n", " graph = (optional_integer + optional_fractional + optional_quantity).optimize()\n", - " self.numbers = graph # Saving just the part of the graph used for numbers\n", + " self.numbers = graph # Saving just the part of the graph used for numbers\n", " graph = optional_sign + graph\n", " delete_tokens = self.delete_tokens(graph)\n", " self.fst = delete_tokens.optimize()" @@ -4254,7 +4332,7 @@ "example2 = 'decimal { integer_part: \"22323\" fractional_part: \"104553\" }'\n", "\n", "apply_fst(example1, fst)\n", - "apply_fst(example2, fst)\n" + "apply_fst(example2, fst)" ] }, { @@ -4356,7 +4434,7 @@ "from nemo_text_processing.inverse_text_normalization.fr.taggers import cardinal\n", "\n", "cardinal_graph = cardinal.CardinalFst()\n", - "graph_cardinal = cardinal_graph.graph_no_exception # graphs cardinals w/o tokenization\n", + "graph_cardinal = cardinal_graph.graph_no_exception # graphs cardinals w/o tokenization\n", "\n", "add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert(\"0\") + NEMO_DIGIT)\n", "graph_fractional_values = graph_cardinal @ add_leading_zero_to_double_digit" @@ -4383,27 +4461,27 @@ " delete_virgule = pynutil.delete(\"virgule\")\n", "\n", " graph_integer = pynutil.insert(\"integer_part: \\\"\") + cardinal + pynutil.insert(\"\\\" \") + delete_space\n", - " graph_integer_or_none = graph_integer | pynutil.insert(\"integer_part: \\\"0\\\" \", weight=.001) # In cases we don't always have an integer preceding\n", + " graph_integer_or_none = graph_integer | pynutil.insert(\n", + " \"integer_part: \\\"0\\\" \", weight=0.001\n", + " ) # In cases we don't always have an integer preceding\n", "\n", " graph_string_of_cardinals = delete_space + cardinal\n", " graph_string_of_cardinals = pynini.closure(graph_string_of_cardinals, 1)\n", " graph_fractional = pynutil.insert(\"fractional_part: \\\"\") + graph_string_of_cardinals + pynutil.insert(\"\\\"\")\n", "\n", - " graph_decimal_no_sign = graph_integer_or_none + delete_virgule + graph_fractional \n", + " graph_decimal_no_sign = graph_integer_or_none + delete_virgule + graph_fractional\n", "\n", " ### NEW GRAPH HERE\n", " self.final_graph_wo_negative = graph_decimal_no_sign | get_quantity(\n", " final_graph_wo_sign, cardinal.graph_hundreds_component_at_least_one_none_zero_digit\n", " )\n", - " \n", + "\n", " graph_negative = pynini.cross(\"moins\", \"negative: \\\"-\\\" \") + delete_space\n", " graph_negative = pynini.closure(graph_negative, 0, 1)\n", " graph_decimal = graph_negative + graph_decimal_no_sign\n", "\n", " # Union default decimal with version that accepts quantities\n", - " graph_decimal |= graph_negative + get_quantity(\n", - " graph_decimal_no_sign, quantities_cardinal\n", - " )\n", + " graph_decimal |= graph_negative + get_quantity(graph_decimal_no_sign, quantities_cardinal)\n", " final_graph = self.add_tokens(graph_decimal)\n", " self.fst = final_graph.optimize()" ] @@ -4426,8 +4504,8 @@ "cardinal_graph = cardinal.CardinalFst()\n", "decimal_graph = decimal.DecimalFst(cardinal_graph)\n", "\n", - "graph_cardinal = cardinal_graph.graph_no_exception # graphs cardinals w/o tokenization\n", - "graph_decimal = decimal_graph.final_graph_wo_negative # graphs positive decimals w/o tokenization\n", + "graph_cardinal = cardinal_graph.graph_no_exception # graphs cardinals w/o tokenization\n", + "graph_decimal = decimal_graph.final_graph_wo_negative # graphs positive decimals w/o tokenization\n", "\n", "add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert(\"0\") + NEMO_DIGIT)\n", "graph_fractional_values = graph_cardinal @ add_leading_zero_to_double_digit" @@ -4473,7 +4551,7 @@ }, "outputs": [], "source": [ - "graph_preposition = pynini.union(\"des \", \"d'\") # Used for large amounts (billions de euros)\n", + "graph_preposition = pynini.union(\"des \", \"d'\") # Used for large amounts (billions de euros)\n", "major_currency_no_normalize = pynini.closure(graph_preposition, 0, 1) + major_currency.project(\"input\")" ] }, @@ -4520,7 +4598,7 @@ " minor_currency += graph_plural\n", "\n", " major_currency_no_normalize = major_currency.project(\"input\")\n", - " graph_preposition = pynini.union(\"des \", \"d'\") # Used for large amounts (billions de euros)\n", + " graph_preposition = pynini.union(\"des \", \"d'\") # Used for large amounts (billions de euros)\n", " major_currency_no_normalize = graph_preposition + major_currency.project(\"input\")\n", "\n", " graph_cardinal = cardinal.graph_no_exception\n", @@ -4574,13 +4652,13 @@ "outputs": [], "source": [ "graph_integer_component = pynutil.insert(\"integer_part: \\\"\") + graph_cardinal + pynutil.insert(\"\\\"\")\n", - "graph_fractional_component = pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", + "graph_fractional_component = pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", "\n", "graph_major_currency = pynutil.insert(\" currency: \\\"\") + major_currency + pynutil.insert(\"\\\"\")\n", "graph_minor_currency = pynutil.insert(\" currency: \\\"\") + minor_currency + pynutil.insert(\"\\\"\")\n", "\n", "graph_only_major_money = graph_integer_component + delete_space + graph_major_currency\n", - "graph_only_minor_money = graph_fractional_component + delete_space + graph_minor_currency " + "graph_only_minor_money = graph_fractional_component + delete_space + graph_minor_currency" ] }, { @@ -4600,8 +4678,10 @@ }, "outputs": [], "source": [ - "implicit_fractional_part = delete_space + pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\") \n", - "implicit_fractional_part = pynini.closure(implicit_fractional_part, 0, 1) " + "implicit_fractional_part = (\n", + " delete_space + pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", + ")\n", + "implicit_fractional_part = pynini.closure(implicit_fractional_part, 0, 1)" ] }, { @@ -4621,12 +4701,12 @@ }, "outputs": [], "source": [ - "delete_et = pynutil.delete(\"et \") # Sometimes prefaces the minor currency\n", - "delete_et = pynini.closure(delete_et, 0 , 1)\n", + "delete_et = pynutil.delete(\"et \") # Sometimes prefaces the minor currency\n", + "delete_et = pynini.closure(delete_et, 0, 1)\n", "\n", - "delete_minor = pynutil.delete(minor_currency.project(\"input\")) # to remove the minor currency\n", + "delete_minor = pynutil.delete(minor_currency.project(\"input\")) # to remove the minor currency\n", "\n", - "explicit_fractional_part = pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\") \n", + "explicit_fractional_part = pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", "explicit_fractional_part = delete_space + delete_et + explicit_fractional_part + delete_space + delete_minor\n", "explicit_fractional_part = pynini.closure(explicit_fractional_part, 0, 1)" ] @@ -4702,7 +4782,7 @@ " minor_currency += graph_plural\n", "\n", " major_currency_no_normalize = major_currency.project(\"input\")\n", - " graph_preposition = pynini.union(\"des \", \"d'\") # Used for large amounts (billions de euros)\n", + " graph_preposition = pynini.union(\"des \", \"d'\") # Used for large amounts (billions de euros)\n", " major_currency_no_normalize = graph_preposition + major_currency.project(\"input\")\n", "\n", " graph_cardinal = cardinal.graph_no_exception\n", @@ -4712,24 +4792,29 @@ " graph_fractional_values = graph_cardinal @ add_leading_zero_to_double_digit\n", "\n", " graph_integer_component = pynutil.insert(\"integer_part: \\\"\") + graph_cardinal + pynutil.insert(\"\\\"\")\n", - " graph_fractional_component = pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", + " graph_fractional_component = (\n", + " pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", + " )\n", "\n", " graph_major_currency = pynutil.insert(\" currency: \\\"\") + major_currency + pynutil.insert(\"\\\"\")\n", " graph_minor_currency = pynutil.insert(\" currency: \\\"\") + minor_currency + pynutil.insert(\"\\\"\")\n", "\n", " graph_only_major_money = graph_integer_component + delete_space + graph_major_currency\n", - " graph_only_minor_money = graph_fractional_component + delete_space + graph_minor_currency \n", - "\n", - " implicit_fractional_part = delete_space + pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\") \n", - " implicit_fractional_part = pynini.closure(implicit_fractional_part, 0, 1) \n", + " graph_only_minor_money = graph_fractional_component + delete_space + graph_minor_currency\n", "\n", + " implicit_fractional_part = (\n", + " delete_space + pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", + " )\n", + " implicit_fractional_part = pynini.closure(implicit_fractional_part, 0, 1)\n", "\n", - " delete_et = pynutil.delete(\"et \") # Sometimes prefaces the minor currency\n", - " delete_et = pynini.closure(delete_et, 0 , 1)\n", + " delete_et = pynutil.delete(\"et \") # Sometimes prefaces the minor currency\n", + " delete_et = pynini.closure(delete_et, 0, 1)\n", "\n", - " delete_minor = pynutil.delete(minor_currency.project(\"input\")) # to remove the minor currency\n", + " delete_minor = pynutil.delete(minor_currency.project(\"input\")) # to remove the minor currency\n", "\n", - " explicit_fractional_part = pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\") \n", + " explicit_fractional_part = (\n", + " pynutil.insert(\"fractional_part: \\\"\") + graph_fractional_values + pynutil.insert(\"\\\"\")\n", + " )\n", " explicit_fractional_part = delete_space + delete_et + explicit_fractional_part + delete_space + delete_minor\n", " explicit_fractional_part = pynini.closure(explicit_fractional_part, 0, 1)\n", "\n", @@ -4857,18 +4942,18 @@ }, "outputs": [], "source": [ - " def __init__(self, decimal: GraphFst):\n", - " super().__init__(name=\"money\", kind=\"verbalize\")\n", - " unit = (\n", - " pynutil.delete(\"currency:\")\n", - " + delete_extra_space\n", - " + pynutil.delete(\"\\\"\")\n", - " + pynini.closure(NEMO_NOT_QUOTE, 1)\n", - " + pynutil.delete(\"\\\"\")\n", - " )\n", - " graph = decimal.numbers + delete_space + unit\n", - " delete_tokens = self.delete_tokens(graph)\n", - " self.fst = delete_tokens.optimize()" + "def __init__(self, decimal: GraphFst):\n", + " super().__init__(name=\"money\", kind=\"verbalize\")\n", + " unit = (\n", + " pynutil.delete(\"currency:\")\n", + " + delete_extra_space\n", + " + pynutil.delete(\"\\\"\")\n", + " + pynini.closure(NEMO_NOT_QUOTE, 1)\n", + " + pynutil.delete(\"\\\"\")\n", + " )\n", + " graph = decimal.numbers + delete_space + unit\n", + " delete_tokens = self.delete_tokens(graph)\n", + " self.fst = delete_tokens.optimize()" ] }, { @@ -4978,99 +5063,103 @@ }, "outputs": [], "source": [ - "hours = pynini.string_map([\n", - " (\"zéro\",\"0\"),\n", - " (\"une\",\"1\"),\n", - " (\"deux\",\"2\"),\n", - " (\"trois\",\"3\"),\n", - " (\"quatre\",\"4\"),\n", - " (\"cinq\",\"5\"),\n", - " (\"six\",\"6\"),\n", - " (\"sept\",\"7\"),\n", - " (\"huit\",\"8\"),\n", - " (\"neuf\",\"9\"),\n", - " (\"dix\",\"10\"),\n", - " (\"onze\",\"11\"),\n", - " (\"douze\",\"12\"),\n", - " (\"treize\",\"13\"),\n", - " (\"quatorze\",\"14\"),\n", - " (\"quinze\",\"15\"),\n", - " (\"seize\",\"16\"),\n", - " (\"dix-sept\",\"17\"),\n", - " (\"dix-huit\",\"18\"),\n", - " (\"dix-neuf\",\"19\"),\n", - " (\"vingt\",\"20\"),\n", - " (\"vingt-et-une\",\"21\"),\n", - " (\"vingt et une\",\"21\"),\n", - " (\"vingt-deux\",\"22\"),\n", - " (\"vingt-trois\",\"23\"),\n", - " (\"vingt-quatre\",\"24\"),\n", - "])\n", - "minutes = pynini.string_map([\n", - " (\"une\", \"01\"),\n", - " (\"deux\", \"02\"),\n", - " (\"trois\", \"03\"),\n", - " (\"quatre\", \"04\"),\n", - " (\"cinq\", \"05\"),\n", - " (\"six\", \"06\"),\n", - " (\"sept\", \"07\"),\n", - " (\"huit\", \"08\"),\n", - " (\"neuf\", \"09\"),\n", - " (\"dix\", \"10\"),\n", - " (\"onze\", \"11\"),\n", - " (\"douze\", \"12\"),\n", - " (\"treize\", \"13\"),\n", - " (\"quatorze\", \"14\"),\n", - " (\"quinze\", \"15\"),\n", - " (\"seize\", \"16\"),\n", - " (\"dix-sept\", \"17\"),\n", - " (\"dix-huit\", \"18\"),\n", - " (\"dix-neuf\", \"19\"),\n", - " (\"vingt\", \"20\"),\n", - " (\"vingt-et-une\", \"21\"),\n", - " (\"vingt et une\", \"21\"),\n", - " (\"vingt-deux\", \"22\"),\n", - " (\"vingt-trois\", \"23\"),\n", - " (\"vingt-quatre\", \"27\"),\n", - " (\"vingt-cinq\", \"25\"),\n", - " (\"vingt-six\", \"26\"),\n", - " (\"vingt-sept\", \"27\"),\n", - " (\"vingt-huit\", \"28\"),\n", - " (\"vingt-neuf\", \"29\"),\n", - " (\"trente\", \"30\"),\n", - " (\"trente-et-une\", \"31\"),\n", - " (\"trente et une\", \"31\"),\n", - " (\"trente-deux\", \"32\"),\n", - " (\"trente-trois\", \"33\"),\n", - " (\"trente-quatre\", \"34\"),\n", - " (\"trente-cinq\", \"35\"),\n", - " (\"trente-six\", \"36\"),\n", - " (\"trente-sept\", \"37\"),\n", - " (\"trente-huit\", \"38\"),\n", - " (\"trente-neuf\", \"39\"),\n", - " (\"quarante\", \"40\"),\n", - " (\"quarante-et-une\", \"41\"),\n", - " (\"quarante et une\", \"41\"),\n", - " (\"quarante-deux\", \"42\"),\n", - " (\"quarante-trois\", \"43\"),\n", - " (\"quarante-quatre\", \"44\"),\n", - " (\"quarante-cinq\", \"45\"),\n", - " (\"quarante-six\", \"46\"),\n", - " (\"quarante-sept\", \"47\"),\n", - " (\"quarante-huit\", \"48\"),\n", - " (\"quarante-neuf\", \"49\"),\n", - " (\"cinquante\", \"50\"),\n", - " (\"cinquante-et-une\", \"51\"),\n", - " (\"cinquante et une\", \"51\"),\n", - " (\"cinquante-deux\", \"52\"),\n", - " (\"cinquante-trois\", \"53\"),\n", - " (\"cinquante-quatre\", \"54\"),\n", - " (\"cinquante-cinq\", \"55\"),\n", - " (\"cinquante-six\", \"56\"),\n", - " (\"cinquante-sept\", \"57\"),\n", - " (\"cinquante-huit\", \"58\"),\n", - " (\"cinquante-neuf\", \"59\"),\n", - "])" + "hours = pynini.string_map(\n", + " [\n", + " (\"zéro\", \"0\"),\n", + " (\"une\", \"1\"),\n", + " (\"deux\", \"2\"),\n", + " (\"trois\", \"3\"),\n", + " (\"quatre\", \"4\"),\n", + " (\"cinq\", \"5\"),\n", + " (\"six\", \"6\"),\n", + " (\"sept\", \"7\"),\n", + " (\"huit\", \"8\"),\n", + " (\"neuf\", \"9\"),\n", + " (\"dix\", \"10\"),\n", + " (\"onze\", \"11\"),\n", + " (\"douze\", \"12\"),\n", + " (\"treize\", \"13\"),\n", + " (\"quatorze\", \"14\"),\n", + " (\"quinze\", \"15\"),\n", + " (\"seize\", \"16\"),\n", + " (\"dix-sept\", \"17\"),\n", + " (\"dix-huit\", \"18\"),\n", + " (\"dix-neuf\", \"19\"),\n", + " (\"vingt\", \"20\"),\n", + " (\"vingt-et-une\", \"21\"),\n", + " (\"vingt et une\", \"21\"),\n", + " (\"vingt-deux\", \"22\"),\n", + " (\"vingt-trois\", \"23\"),\n", + " (\"vingt-quatre\", \"24\"),\n", + " ]\n", + ")\n", + "minutes = pynini.string_map(\n", + " [\n", + " (\"une\", \"01\"),\n", + " (\"deux\", \"02\"),\n", + " (\"trois\", \"03\"),\n", + " (\"quatre\", \"04\"),\n", + " (\"cinq\", \"05\"),\n", + " (\"six\", \"06\"),\n", + " (\"sept\", \"07\"),\n", + " (\"huit\", \"08\"),\n", + " (\"neuf\", \"09\"),\n", + " (\"dix\", \"10\"),\n", + " (\"onze\", \"11\"),\n", + " (\"douze\", \"12\"),\n", + " (\"treize\", \"13\"),\n", + " (\"quatorze\", \"14\"),\n", + " (\"quinze\", \"15\"),\n", + " (\"seize\", \"16\"),\n", + " (\"dix-sept\", \"17\"),\n", + " (\"dix-huit\", \"18\"),\n", + " (\"dix-neuf\", \"19\"),\n", + " (\"vingt\", \"20\"),\n", + " (\"vingt-et-une\", \"21\"),\n", + " (\"vingt et une\", \"21\"),\n", + " (\"vingt-deux\", \"22\"),\n", + " (\"vingt-trois\", \"23\"),\n", + " (\"vingt-quatre\", \"27\"),\n", + " (\"vingt-cinq\", \"25\"),\n", + " (\"vingt-six\", \"26\"),\n", + " (\"vingt-sept\", \"27\"),\n", + " (\"vingt-huit\", \"28\"),\n", + " (\"vingt-neuf\", \"29\"),\n", + " (\"trente\", \"30\"),\n", + " (\"trente-et-une\", \"31\"),\n", + " (\"trente et une\", \"31\"),\n", + " (\"trente-deux\", \"32\"),\n", + " (\"trente-trois\", \"33\"),\n", + " (\"trente-quatre\", \"34\"),\n", + " (\"trente-cinq\", \"35\"),\n", + " (\"trente-six\", \"36\"),\n", + " (\"trente-sept\", \"37\"),\n", + " (\"trente-huit\", \"38\"),\n", + " (\"trente-neuf\", \"39\"),\n", + " (\"quarante\", \"40\"),\n", + " (\"quarante-et-une\", \"41\"),\n", + " (\"quarante et une\", \"41\"),\n", + " (\"quarante-deux\", \"42\"),\n", + " (\"quarante-trois\", \"43\"),\n", + " (\"quarante-quatre\", \"44\"),\n", + " (\"quarante-cinq\", \"45\"),\n", + " (\"quarante-six\", \"46\"),\n", + " (\"quarante-sept\", \"47\"),\n", + " (\"quarante-huit\", \"48\"),\n", + " (\"quarante-neuf\", \"49\"),\n", + " (\"cinquante\", \"50\"),\n", + " (\"cinquante-et-une\", \"51\"),\n", + " (\"cinquante et une\", \"51\"),\n", + " (\"cinquante-deux\", \"52\"),\n", + " (\"cinquante-trois\", \"53\"),\n", + " (\"cinquante-quatre\", \"54\"),\n", + " (\"cinquante-cinq\", \"55\"),\n", + " (\"cinquante-six\", \"56\"),\n", + " (\"cinquante-sept\", \"57\"),\n", + " (\"cinquante-huit\", \"58\"),\n", + " (\"cinquante-neuf\", \"59\"),\n", + " ]\n", + ")" ] }, { @@ -5164,94 +5253,98 @@ }, "outputs": [], "source": [ - "hours_to = pynini.string_map([\n", - " (\"1\",\"0\"),\n", - " (\"2\",\"1\"),\n", - " (\"3\",\"2\"),\n", - " (\"4\",\"3\"),\n", - " (\"5\",\"4\"),\n", - " (\"6\",\"5\"),\n", - " (\"7\",\"6\"),\n", - " (\"8\",\"7\"),\n", - " (\"9\",\"8\"),\n", - " (\"10\",\"9\"),\n", - " (\"11\",\"10\"),\n", - " (\"12\",\"11\"),\n", - " (\"13\",\"12\"),\n", - " (\"14\",\"13\"),\n", - " (\"15\",\"14\"),\n", - " (\"16\",\"15\"),\n", - " (\"17\",\"16\"),\n", - " (\"18\",\"17\"),\n", - " (\"19\",\"18\"),\n", - " (\"20\",\"19\"),\n", - " (\"21\",\"20\"),\n", - " (\"22\",\"21\"),\n", - " (\"23\",\"22\"),\n", - " (\"24\",\"23\"),\n", - " (\"0\",\"23\"),\n", - "])\n", - "minutes_to = pynini.string_map([\n", - " (\"59\", \"01\"),\n", - " (\"58\", \"02\"),\n", - " (\"57\", \"03\"),\n", - " (\"56\", \"04\"),\n", - " (\"55\", \"05\"),\n", - " (\"54\", \"06\"),\n", - " (\"53\", \"07\"),\n", - " (\"52\", \"08\"),\n", - " (\"51\", \"09\"),\n", - " (\"50\", \"10\"),\n", - " (\"49\", \"11\"),\n", - " (\"48\", \"12\"),\n", - " (\"47\", \"13\"),\n", - " (\"46\", \"14\"),\n", - " (\"45\", \"15\"),\n", - " (\"44\", \"16\"),\n", - " (\"43\", \"17\"),\n", - " (\"42\", \"18\"),\n", - " (\"41\", \"19\"),\n", - " (\"40\", \"20\"),\n", - " (\"39\", \"21\"),\n", - " (\"38\", \"22\"),\n", - " (\"37\", \"23\"),\n", - " (\"36\", \"24\"),\n", - " (\"35\", \"25\"),\n", - " (\"34\", \"26\"),\n", - " (\"33\", \"27\"),\n", - " (\"32\", \"28\"),\n", - " (\"31\", \"29\"),\n", - " (\"30\", \"30\"),\n", - " (\"29\", \"31\"),\n", - " (\"28\", \"32\"),\n", - " (\"27\", \"33\"),\n", - " (\"26\", \"34\"),\n", - " (\"25\", \"35\"),\n", - " (\"24\", \"36\"),\n", - " (\"23\", \"37\"),\n", - " (\"22\", \"38\"),\n", - " (\"21\", \"39\"),\n", - " (\"20\", \"40\"),\n", - " (\"19\", \"41\"),\n", - " (\"18\", \"42\"),\n", - " (\"17\", \"43\"),\n", - " (\"16\", \"44\"),\n", - " (\"15\", \"45\"),\n", - " (\"14\", \"46\"),\n", - " (\"13\", \"47\"),\n", - " (\"12\", \"48\"),\n", - " (\"11\", \"49\"),\n", - " (\"10\", \"50\"),\n", - " (\"09\", \"51\"),\n", - " (\"08\", \"52\"),\n", - " (\"07\", \"53\"),\n", - " (\"06\", \"54\"),\n", - " (\"05\", \"55\"),\n", - " (\"04\", \"56\"),\n", - " (\"03\", \"57\"),\n", - " (\"02\", \"58\"),\n", - " (\"01\", \"59\"),\n", - "])\n", + "hours_to = pynini.string_map(\n", + " [\n", + " (\"1\", \"0\"),\n", + " (\"2\", \"1\"),\n", + " (\"3\", \"2\"),\n", + " (\"4\", \"3\"),\n", + " (\"5\", \"4\"),\n", + " (\"6\", \"5\"),\n", + " (\"7\", \"6\"),\n", + " (\"8\", \"7\"),\n", + " (\"9\", \"8\"),\n", + " (\"10\", \"9\"),\n", + " (\"11\", \"10\"),\n", + " (\"12\", \"11\"),\n", + " (\"13\", \"12\"),\n", + " (\"14\", \"13\"),\n", + " (\"15\", \"14\"),\n", + " (\"16\", \"15\"),\n", + " (\"17\", \"16\"),\n", + " (\"18\", \"17\"),\n", + " (\"19\", \"18\"),\n", + " (\"20\", \"19\"),\n", + " (\"21\", \"20\"),\n", + " (\"22\", \"21\"),\n", + " (\"23\", \"22\"),\n", + " (\"24\", \"23\"),\n", + " (\"0\", \"23\"),\n", + " ]\n", + ")\n", + "minutes_to = pynini.string_map(\n", + " [\n", + " (\"59\", \"01\"),\n", + " (\"58\", \"02\"),\n", + " (\"57\", \"03\"),\n", + " (\"56\", \"04\"),\n", + " (\"55\", \"05\"),\n", + " (\"54\", \"06\"),\n", + " (\"53\", \"07\"),\n", + " (\"52\", \"08\"),\n", + " (\"51\", \"09\"),\n", + " (\"50\", \"10\"),\n", + " (\"49\", \"11\"),\n", + " (\"48\", \"12\"),\n", + " (\"47\", \"13\"),\n", + " (\"46\", \"14\"),\n", + " (\"45\", \"15\"),\n", + " (\"44\", \"16\"),\n", + " (\"43\", \"17\"),\n", + " (\"42\", \"18\"),\n", + " (\"41\", \"19\"),\n", + " (\"40\", \"20\"),\n", + " (\"39\", \"21\"),\n", + " (\"38\", \"22\"),\n", + " (\"37\", \"23\"),\n", + " (\"36\", \"24\"),\n", + " (\"35\", \"25\"),\n", + " (\"34\", \"26\"),\n", + " (\"33\", \"27\"),\n", + " (\"32\", \"28\"),\n", + " (\"31\", \"29\"),\n", + " (\"30\", \"30\"),\n", + " (\"29\", \"31\"),\n", + " (\"28\", \"32\"),\n", + " (\"27\", \"33\"),\n", + " (\"26\", \"34\"),\n", + " (\"25\", \"35\"),\n", + " (\"24\", \"36\"),\n", + " (\"23\", \"37\"),\n", + " (\"22\", \"38\"),\n", + " (\"21\", \"39\"),\n", + " (\"20\", \"40\"),\n", + " (\"19\", \"41\"),\n", + " (\"18\", \"42\"),\n", + " (\"17\", \"43\"),\n", + " (\"16\", \"44\"),\n", + " (\"15\", \"45\"),\n", + " (\"14\", \"46\"),\n", + " (\"13\", \"47\"),\n", + " (\"12\", \"48\"),\n", + " (\"11\", \"49\"),\n", + " (\"10\", \"50\"),\n", + " (\"09\", \"51\"),\n", + " (\"08\", \"52\"),\n", + " (\"07\", \"53\"),\n", + " (\"06\", \"54\"),\n", + " (\"05\", \"55\"),\n", + " (\"04\", \"56\"),\n", + " (\"03\", \"57\"),\n", + " (\"02\", \"58\"),\n", + " (\"01\", \"59\"),\n", + " ]\n", + ")\n", "graph_moins = pynutil.delete(\"moins\")" ] }, @@ -5300,11 +5393,12 @@ "\n", "graph_minutes_component = (\n", " pynutil.insert(\" minutes: \\\"\") + pynini.union(minutes, graph_fractions) + pynutil.insert(\"\\\"\")\n", - ") \n", + ")\n", "graph_minutes_component = delete_space + graph_minutes_component\n", "\n", - "graph_time_standard = (graph_hours_component + delete_space + graph_heures \n", - " + pynini.closure(graph_minutes_component, 0, 1))" + "graph_time_standard = (\n", + " graph_hours_component + delete_space + graph_heures + pynini.closure(graph_minutes_component, 0, 1)\n", + ")" ] }, { @@ -5329,8 +5423,9 @@ "graph_hours_to_component = pynutil.insert(\"hours: \\\"\") + graph_hours_to_component + pynutil.insert(\"\\\"\")\n", "graph_hours_to_component = graph_hours_to_component + delete_space + graph_heures\n", "\n", - "graph_minutes_to_component = (minutes | graph_demi | # No 'et' in fractions\n", - " (pynutil.delete(\"le \") + graph_quart) | graph_trois_quart)\n", + "graph_minutes_to_component = (\n", + " minutes | graph_demi | (pynutil.delete(\"le \") + graph_quart) | graph_trois_quart # No 'et' in fractions\n", + ")\n", "graph_minutes_to_component @= minutes_to\n", "graph_minutes_to_component = pynutil.insert(\" minutes: \\\"\") + graph_minutes_to_component + pynutil.insert(\"\\\"\")\n", "\n", @@ -5381,7 +5476,7 @@ "outputs": [], "source": [ "graph_suffix_am = pynini.cross(\"du matin\", \"am\")\n", - "graph_suffix_pm = pynini.string_map([(\"de l'après-midi\", \"pm\"),(\"du soir\", \"pm\")])\n", + "graph_suffix_pm = pynini.string_map([(\"de l'après-midi\", \"pm\"), (\"du soir\", \"pm\")])\n", "\n", "graph_suffix = pynini.cross(graph_suffix_am, \"am\") | pynini.cross(graph_suffix_pm, \"pm\")\n", "\n", @@ -5420,40 +5515,44 @@ "\n", " graph_minutes_component = (\n", " pynutil.insert(\" minutes: \\\"\") + pynini.union(minutes, graph_fractions) + pynutil.insert(\"\\\"\")\n", - " ) \n", + " )\n", " graph_minutes_component = delete_space + graph_minutes_component\n", "\n", - " graph_time_standard = (graph_hours_component + delete_space + graph_heures \n", - " + pynini.closure(graph_minutes_component, 0, 1))\n", + " graph_time_standard = (\n", + " graph_hours_component + delete_space + graph_heures + pynini.closure(graph_minutes_component, 0, 1)\n", + " )\n", "\n", " graph_hours_to_component = hours | graph_midi | graph_minuit\n", " graph_hours_to_component @= hours_to\n", " graph_hours_to_component = pynutil.insert(\"hours: \\\"\") + graph_hours_to_component + pynutil.insert(\"\\\"\")\n", " graph_hours_to_component = graph_hours_to_component + delete_space + graph_heures\n", "\n", - " graph_minutes_to_component = (minutes | graph_demi | # No 'et' in fractions\n", - " (pynutil.delete(\"le \") + graph_quart) | graph_trois_quart)\n", + " graph_minutes_to_component = (\n", + " minutes | graph_demi | (pynutil.delete(\"le \") + graph_quart) | graph_trois_quart # No 'et' in fractions\n", + " )\n", " graph_minutes_to_component @= minutes_to\n", " graph_minutes_to_component = pynutil.insert(\" minutes: \\\"\") + graph_minutes_to_component + pynutil.insert(\"\\\"\")\n", "\n", - " graph_time_to = graph_hours_to_component + delete_space + graph_moins + delete_space + graph_minutes_to_component\n", + " graph_time_to = (\n", + " graph_hours_to_component + delete_space + graph_moins + delete_space + graph_minutes_to_component\n", + " )\n", "\n", " graph_time_no_suffix = graph_time_standard | graph_time_to\n", "\n", " graph_suffix_am = pynini.cross(\"du matin\", \"am\")\n", - " graph_suffix_pm = pynini.string_map([(\"de l'après-midi\", \"pm\"),(\"du soir\", \"pm\")])\n", + " graph_suffix_pm = pynini.string_map([(\"de l'après-midi\", \"pm\"), (\"du soir\", \"pm\")])\n", "\n", " graph_suffix = pynini.cross(graph_suffix_am, \"am\") | pynini.cross(graph_suffix_pm, \"pm\")\n", "\n", " graph_suffix_component = pynutil.insert(\" suffix: \\\"\") + graph_suffix + pynutil.insert(\"\\\"\")\n", " graph_suffix_component = delete_space + graph_suffix_component\n", " graph_suffix_component = pynini.closure(graph_suffix_component, 0, 1)\n", - " \n", + "\n", " final_graph = graph_time_no_suffix + graph_suffix_component\n", "\n", " final_graph = self.add_tokens(final_graph)\n", "\n", - " self.fst = final_graph.optimize()\n" + " self.fst = final_graph.optimize()" ] }, { @@ -5556,19 +5655,21 @@ }, "outputs": [], "source": [ - "hour_to_night = pynini.string_map([\n", - " (\"1\", \"13\"),\n", - " (\"2\", \"14\"),\n", - " (\"3\", \"15\"),\n", - " (\"4\", \"16\"),\n", - " (\"5\", \"17\"),\n", - " (\"6\", \"18\"),\n", - " (\"7\", \"19\"),\n", - " (\"8\", \"20\"),\n", - " (\"9\", \"21\"),\n", - " (\"10\", \"22\"),\n", - " (\"11\", \"23\"), # Note that 12 and 24 would be phrased \"midi\" and \"minuit\" respectively\n", - "])" + "hour_to_night = pynini.string_map(\n", + " [\n", + " (\"1\", \"13\"),\n", + " (\"2\", \"14\"),\n", + " (\"3\", \"15\"),\n", + " (\"4\", \"16\"),\n", + " (\"5\", \"17\"),\n", + " (\"6\", \"18\"),\n", + " (\"7\", \"19\"),\n", + " (\"8\", \"20\"),\n", + " (\"9\", \"21\"),\n", + " (\"10\", \"22\"),\n", + " (\"11\", \"23\"), # Note that 12 and 24 would be phrased \"midi\" and \"minuit\" respectively\n", + " ]\n", + ")" ] }, { @@ -5589,14 +5690,7 @@ "outputs": [], "source": [ "night_suffixes = pynutil.delete(\"suffix: \\\"pm\\\"\")\n", - "graph |= (\n", - " hour @ hour_to_night\n", - " + delete_extra_space\n", - " + pynutil.insert(\"h\")\n", - " + minute.ques\n", - " + delete_space\n", - " + night_suffixes\n", - " )" + "graph |= hour @ hour_to_night + delete_extra_space + pynutil.insert(\"h\") + minute.ques + delete_space + night_suffixes" ] }, { @@ -5620,19 +5714,21 @@ " def __init__(self):\n", " super().__init__(name=\"time\", kind=\"verbalize\")\n", "\n", - " hour_to_night = pynini.string_map([\n", - " (\"1\", \"13\"),\n", - " (\"2\", \"14\"),\n", - " (\"3\", \"15\"),\n", - " (\"4\", \"16\"),\n", - " (\"5\", \"17\"),\n", - " (\"6\", \"18\"),\n", - " (\"7\", \"19\"),\n", - " (\"8\", \"20\"),\n", - " (\"9\", \"21\"),\n", - " (\"10\", \"22\"),\n", - " (\"11\", \"23\"),\n", - "])\n", + " hour_to_night = pynini.string_map(\n", + " [\n", + " (\"1\", \"13\"),\n", + " (\"2\", \"14\"),\n", + " (\"3\", \"15\"),\n", + " (\"4\", \"16\"),\n", + " (\"5\", \"17\"),\n", + " (\"6\", \"18\"),\n", + " (\"7\", \"19\"),\n", + " (\"8\", \"20\"),\n", + " (\"9\", \"21\"),\n", + " (\"10\", \"22\"),\n", + " (\"11\", \"23\"),\n", + " ]\n", + " )\n", "\n", " day_suffixes = pynutil.delete(\"suffix: \\\"am\\\"\")\n", " night_suffixes = pynutil.delete(\"suffix: \\\"pm\\\"\")\n", @@ -5723,11 +5819,7 @@ }, "outputs": [], "source": [ - "graph = pynini.string_map([\n", - " (\"mister\", \"mr.\"),\n", - " (\"h m s\", \"h.m.s\"),\n", - " (\"doctor\", \"dr.\")\n", - "])" + "graph = pynini.string_map([(\"mister\", \"mr.\"), (\"h m s\", \"h.m.s\"), (\"doctor\", \"dr.\")])" ] }, { @@ -5769,10 +5861,7 @@ " def __init__(self):\n", " super().__init__(name=\"whitelist\", kind=\"classify\")\n", "\n", - " whitelist = pynini.string_map([\n", - " (\"mister\", \"mr.\"),\n", - " (\"h m s\", \"h.m.s\"),\n", - " (\"doctor\", \"dr.\")])\n", + " whitelist = pynini.string_map([(\"mister\", \"mr.\"), (\"h m s\", \"h.m.s\"), (\"doctor\", \"dr.\")])\n", " graph = pynutil.insert(\"name: \\\"\") + convert_space(whitelist) + pynutil.insert(\"\\\"\")\n", " self.fst = graph.optimize()" ] @@ -5811,7 +5900,9 @@ " + pynini.closure(NEMO_CHAR - \" \", 1)\n", " + pynutil.delete(\"\\\"\")\n", " )\n", - " graph = graph @ pynini.cdrewrite(pynini.cross(u\"\\u00A0\", \" \"), \"\", \"\", NEMO_SIGMA) # Removes possible null token\n", + " graph = graph @ pynini.cdrewrite(\n", + " pynini.cross(u\"\\u00a0\", \" \"), \"\", \"\", NEMO_SIGMA\n", + " ) # Removes possible null token\n", " self.fst = graph.optimize()" ] }, @@ -5932,7 +6023,7 @@ " super().__init__(name=\"punctuation\", kind=\"classify\")\n", "\n", " s = \"!#$%&\\'()*+,-./:;<=>?@^_`{|}~\"\n", - " guillemets = \"\\u00AB\" + \"\\u00BB\" # quotation marks in French.\n", + " guillemets = \"\\u00ab\" + \"\\u00bb\" # quotation marks in French.\n", " s += guillemets\n", " punct = pynini.union(*s)\n", "\n", @@ -5972,7 +6063,9 @@ " super().__init__(name=\"word\", kind=\"verbalize\")\n", " chars = pynini.closure(NEMO_CHAR - \" \", 1)\n", " char = pynutil.delete(\"name:\") + delete_space + pynutil.delete(\"\\\"\") + chars + pynutil.delete(\"\\\"\")\n", - " graph = char @ pynini.cdrewrite(pynini.cross(u\"\\u00A0\", \" \"), \"\", \"\", NEMO_SIGMA) # Cleans up possible null character\n", + " graph = char @ pynini.cdrewrite(\n", + " pynini.cross(u\"\\u00a0\", \" \"), \"\", \"\", NEMO_SIGMA\n", + " ) # Cleans up possible null character\n", "\n", " self.fst = graph.optimize()" ] @@ -6167,13 +6260,7 @@ " punct_graph = PunctuationFst().fst\n", "\n", " classify = (\n", - " time_graph\n", - " | whitelist_graph\n", - " | decimal_graph\n", - " | cardinal_graph\n", - " | ordinal_graph\n", - " | money_graph\n", - " | word_graph\n", + " time_graph | whitelist_graph | decimal_graph | cardinal_graph | ordinal_graph | money_graph | word_graph\n", " )\n", " token = pynutil.insert(\"tokens { \") + classify + pynutil.insert(\" }\")" ] @@ -6220,9 +6307,9 @@ "source": [ "token = \"PLACEHOLDER\"\n", "token_plus_punct = (\n", - " pynini.closure(punct + pynutil.insert(\" \")) + token + pynini.closure(pynutil.insert(\" \") + punct)\n", - " ) # Note the use of closure incase there are multiple punctuations\n", - "graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)\n" + " pynini.closure(punct + pynutil.insert(\" \")) + token + pynini.closure(pynutil.insert(\" \") + punct)\n", + ") # Note the use of closure incase there are multiple punctuations\n", + "graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)" ] }, { @@ -6273,15 +6360,7 @@ }, "outputs": [], "source": [ - "classify = (\n", - " time_graph\n", - " | whitelist_graph\n", - " | decimal_graph\n", - " | cardinal_graph\n", - " | ordinal_graph\n", - " | money_graph\n", - " | word_graph\n", - " )\n", + "classify = time_graph | whitelist_graph | decimal_graph | cardinal_graph | ordinal_graph | money_graph | word_graph\n", "punct = pynutil.insert(\"tokens { \") + punct_graph + pynutil.insert(\" }\")" ] }, @@ -6303,14 +6382,14 @@ "outputs": [], "source": [ "classify = (\n", - " pynutil.add_weight(time_graph, 1)\n", - " | pynutil.add_weight(whitelist_graph, 1)\n", - " | pynutil.add_weight(decimal_graph, 1)\n", - " | pynutil.add_weight(cardinal_graph, 1)\n", - " | pynutil.add_weight(ordinal_graph, 1)\n", - " | pynutil.add_weight(money_graph, 1)\n", - " | pynutil.add_weight(word_graph, 1)\n", - " )\n", + " pynutil.add_weight(time_graph, 1)\n", + " | pynutil.add_weight(whitelist_graph, 1)\n", + " | pynutil.add_weight(decimal_graph, 1)\n", + " | pynutil.add_weight(cardinal_graph, 1)\n", + " | pynutil.add_weight(ordinal_graph, 1)\n", + " | pynutil.add_weight(money_graph, 1)\n", + " | pynutil.add_weight(word_graph, 1)\n", + ")\n", "punct = pynutil.insert(\"tokens { \") + pynutil.add_weight(punct_graph, 1) + pynutil.insert(\" }\")" ] }, @@ -6334,14 +6413,14 @@ "outputs": [], "source": [ "classify = (\n", - " pynutil.add_weight(time_graph, 1.1)\n", - " | pynutil.add_weight(whitelist_graph, 1.1)\n", - " | pynutil.add_weight(decimal_graph, 1.1)\n", - " | pynutil.add_weight(cardinal_graph, 1.1)\n", - " | pynutil.add_weight(ordinal_graph, 1.1)\n", - " | pynutil.add_weight(money_graph, 1.1)\n", - " | pynutil.add_weight(word_graph, 1.1)\n", - " )\n", + " pynutil.add_weight(time_graph, 1.1)\n", + " | pynutil.add_weight(whitelist_graph, 1.1)\n", + " | pynutil.add_weight(decimal_graph, 1.1)\n", + " | pynutil.add_weight(cardinal_graph, 1.1)\n", + " | pynutil.add_weight(ordinal_graph, 1.1)\n", + " | pynutil.add_weight(money_graph, 1.1)\n", + " | pynutil.add_weight(word_graph, 1.1)\n", + ")\n", "punct = pynutil.insert(\"tokens { \") + pynutil.add_weight(punct_graph, 1.1) + pynutil.insert(\" }\")" ] }, @@ -6361,14 +6440,14 @@ "outputs": [], "source": [ "classify = (\n", - " pynutil.add_weight(time_graph, 1.1)\n", - " | pynutil.add_weight(whitelist_graph, 1.1)\n", - " | pynutil.add_weight(decimal_graph, 1.1)\n", - " | pynutil.add_weight(cardinal_graph, 1.1)\n", - " | pynutil.add_weight(ordinal_graph, 1.1)\n", - " | pynutil.add_weight(money_graph, 1.1)\n", - " | pynutil.add_weight(word_graph, 100)\n", - " )\n", + " pynutil.add_weight(time_graph, 1.1)\n", + " | pynutil.add_weight(whitelist_graph, 1.1)\n", + " | pynutil.add_weight(decimal_graph, 1.1)\n", + " | pynutil.add_weight(cardinal_graph, 1.1)\n", + " | pynutil.add_weight(ordinal_graph, 1.1)\n", + " | pynutil.add_weight(money_graph, 1.1)\n", + " | pynutil.add_weight(word_graph, 100)\n", + ")\n", "punct = pynutil.insert(\"tokens { \") + pynutil.add_weight(punct_graph, 1.1) + pynutil.insert(\" }\")" ] }, @@ -6397,14 +6476,14 @@ "outputs": [], "source": [ "classify = (\n", - " pynutil.add_weight(time_graph, 1.1)\n", - " | pynutil.add_weight(whitelist_graph, 1.1)\n", - " | pynutil.add_weight(decimal_graph, 1.1)\n", - " | pynutil.add_weight(cardinal_graph, 1.2)\n", - " | pynutil.add_weight(ordinal_graph, 1.1)\n", - " | pynutil.add_weight(money_graph, 1.1)\n", - " | pynutil.add_weight(word_graph, 100)\n", - " )\n", + " pynutil.add_weight(time_graph, 1.1)\n", + " | pynutil.add_weight(whitelist_graph, 1.1)\n", + " | pynutil.add_weight(decimal_graph, 1.1)\n", + " | pynutil.add_weight(cardinal_graph, 1.2)\n", + " | pynutil.add_weight(ordinal_graph, 1.1)\n", + " | pynutil.add_weight(money_graph, 1.1)\n", + " | pynutil.add_weight(word_graph, 100)\n", + ")\n", "punct = pynutil.insert(\"tokens { \") + pynutil.add_weight(punct_graph, 1.1) + pynutil.insert(\" }\")" ] }, @@ -6426,14 +6505,14 @@ "outputs": [], "source": [ "classify = (\n", - " pynutil.add_weight(time_graph, 1)\n", - " | pynutil.add_weight(whitelist_graph, 1)\n", - " | pynutil.add_weight(decimal_graph, 1.1)\n", - " | pynutil.add_weight(cardinal_graph, 1.2)\n", - " | pynutil.add_weight(ordinal_graph, 1)\n", - " | pynutil.add_weight(money_graph, 1.09)\n", - " | pynutil.add_weight(word_graph, 100)\n", - " )\n", + " pynutil.add_weight(time_graph, 1)\n", + " | pynutil.add_weight(whitelist_graph, 1)\n", + " | pynutil.add_weight(decimal_graph, 1.1)\n", + " | pynutil.add_weight(cardinal_graph, 1.2)\n", + " | pynutil.add_weight(ordinal_graph, 1)\n", + " | pynutil.add_weight(money_graph, 1.09)\n", + " | pynutil.add_weight(word_graph, 100)\n", + ")\n", "punct = pynutil.insert(\"tokens { \") + pynutil.add_weight(punct_graph, 1) + pynutil.insert(\" }\")" ] }, @@ -6457,14 +6536,14 @@ "outputs": [], "source": [ "classify = (\n", - " pynutil.add_weight(time_graph, 1.1)\n", - " | pynutil.add_weight(whitelist_graph, 1.07)\n", - " | pynutil.add_weight(decimal_graph, 1.1)\n", - " | pynutil.add_weight(cardinal_graph, 1.2)\n", - " | pynutil.add_weight(ordinal_graph, 1.1)\n", - " | pynutil.add_weight(money_graph, 1.08)\n", - " | pynutil.add_weight(word_graph, 100)\n", - " )\n", + " pynutil.add_weight(time_graph, 1.1)\n", + " | pynutil.add_weight(whitelist_graph, 1.07)\n", + " | pynutil.add_weight(decimal_graph, 1.1)\n", + " | pynutil.add_weight(cardinal_graph, 1.2)\n", + " | pynutil.add_weight(ordinal_graph, 1.1)\n", + " | pynutil.add_weight(money_graph, 1.08)\n", + " | pynutil.add_weight(word_graph, 100)\n", + ")\n", "punct = pynutil.insert(\"tokens { \") + pynutil.add_weight(punct_graph, 1.1) + pynutil.insert(\" }\")" ] }, @@ -6490,7 +6569,7 @@ "class ClassifyFst(GraphFst):\n", " \"\"\"\n", " Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.\n", - " For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File. \n", + " For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.\n", " More details to deployment at NeMo/tools/text_processing_deployment.\n", "\n", " Args:\n", @@ -6614,15 +6693,16 @@ "source": [ "import os\n", "\n", + "\n", "class ClassifyFst(GraphFst):\n", " def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):\n", " super().__init__(name=\"tokenize_and_classify\", kind=\"classify\")\n", " # Grammar here\n", " # ....\n", " if cache_dir is not None and cache_dir != \"None\":\n", - " os.makedirs(cache_dir, exist_ok=True)\n", - " far_file = os.path.join(cache_dir, \"_fr_itn.far\")\n", - " generator_main(far_file, {\"tokenize_and_classify\": self.fst})" + " os.makedirs(cache_dir, exist_ok=True)\n", + " far_file = os.path.join(cache_dir, \"_fr_itn.far\")\n", + " generator_main(far_file, {\"tokenize_and_classify\": self.fst})" ] }, { @@ -6644,6 +6724,7 @@ "source": [ "import os\n", "\n", + "\n", "class ClassifyFst(GraphFst):\n", " def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):\n", " super().__init__(name=\"tokenize_and_classify\", kind=\"classify\")\n", @@ -6655,7 +6736,7 @@ " if cache_dir is not None and cache_dir != \"None\":\n", " os.makedirs(cache_dir, exist_ok=True)\n", " far_file = os.path.join(cache_dir, \"_fr_itn.far\")\n", - " generator_main(far_file, {\"tokenize_and_classify\": self.fst})\n" + " generator_main(far_file, {\"tokenize_and_classify\": self.fst})" ] }, { @@ -6793,6 +6874,7 @@ "from nemo_text_processing.inverse_text_normalization.fr.verbalizers.whitelist import WhiteListFst\n", "from nemo_text_processing.inverse_text_normalization.fr.verbalizers.word import WordFst\n", "\n", + "\n", "class VerbalizeFst(GraphFst):\n", " def __init__(self):\n", " super().__init__(name=\"verbalize\", kind=\"verbalize\")\n", @@ -6804,14 +6886,7 @@ " whitelist_graph = WhiteListFst().fst\n", " money_graph = MoneyFst(decimal=decimal).fst\n", " time_graph = TimeFst().fst\n", - " graph = (\n", - " time_graph\n", - " | whitelist_graph\n", - " | money_graph\n", - " | ordinal_graph\n", - " | decimal_graph\n", - " | cardinal_graph\n", - " )\n", + " graph = time_graph | whitelist_graph | money_graph | ordinal_graph | decimal_graph | cardinal_graph\n", " self.fst = graph" ] }, @@ -6843,7 +6918,6 @@ }, "outputs": [], "source": [ - "\n", "class VerbalizeFinalFst(GraphFst):\n", " def __init__(self):\n", " super().__init__(name=\"verbalize_final\", kind=\"verbalize\")\n",