diff --git a/.github/workflows/check-mo-freshness.yml b/.github/workflows/check-mo-freshness.yml new file mode 100644 index 00000000..53d46ebb --- /dev/null +++ b/.github/workflows/check-mo-freshness.yml @@ -0,0 +1,58 @@ +name: Check .mo bundles are up to date + +on: + workflow_dispatch: + pull_request: + branches: + - development + - main + paths: + - 'tabcmd/locales/**/*.properties' + - 'tabcmd/locales/**/*.mo' + - 'bin/i18n/**' + - 'dodo.py' + - '.github/workflows/check-mo-freshness.yml' + +permissions: + contents: read + +jobs: + check-mo-freshness: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Install localization deps + run: | + python -m pip install --upgrade pip + pip install .[localize] + + - name: Regenerate .po and .mo bundles + run: python -m doit localize + + - name: Verify regenerated files match what is committed + id: verify + run: | + # .po files are globally gitignored, so only check .mo drift. + if ! git diff --exit-code -- 'tabcmd/locales/**/*.mo'; then + echo "" + echo "The committed .mo files are out of date with the .properties sources." + echo "Run 'python -m doit localize' locally and commit the regenerated files." + echo "The freshly-generated files are attached as a workflow artifact." + exit 1 + fi + + # If the verify step failed the regenerated files are still on disk. Uploading + # them makes it easy to download from the failed CI run and commit them. + - name: Upload regenerated .mo files on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: regenerated-mo-files + path: tabcmd/locales/**/*.mo diff --git a/.gitignore b/.gitignore index e37ef3a4..3e094da4 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,8 @@ test-output/ # localization intermediate files *.po +tabcmd/locales/**/combined.tmp +tabcmd/locales/codestrings.tmp # venv site-packages diff --git a/bin/i18n/README.md b/bin/i18n/README.md index 7bac1758..21e25eac 100644 --- a/bin/i18n/README.md +++ b/bin/i18n/README.md @@ -1,28 +1,34 @@ ### Strings and localization -New text should not be hardcoded into the python, but added to tabcmd/locales/en/extra.properties. This file can be given to the translation team and they will return a copy for each other language. Until then, the english string will be used as a fallback. +New text should not be hardcoded into the python, but added to `tabcmd/locales/en/extra.properties` (or directly to `tabcmd/locales/en/tabcmd_messages_en.properties`). The English properties file is given to the translation team, and they return a translated copy for each supported locale. Until translations return, non-English users automatically see the English string via per-key fallback in gettext (see `tabcmd/execution/localize.py`). -To handle localizing text we used the python standard library tool [gettext](https://docs.python.org/3/library/gettext.html). This expects .mo files. I couldn't find a tool that transformed .properties -> .mo directly so we go through .po format. -(FYI: to read mo files for debugging use https://poedit.net/download) +To handle localizing text we use the python standard library tool [gettext](https://docs.python.org/3/library/gettext.html). This expects .mo files. There is no direct .properties -> .mo tool so we go through .po format. +(FYI: to read .mo files for debugging use https://poedit.net/download) +Missing string keys are validated on every PR by `.github/workflows/check-strings.yml`, which runs `python bin/i18n/check_strings.py`. Run it locally the same way before pushing. -These steps are separated for easier troubleshooting: each step is idempotent and will overwrite the existing output. More details about implementation are in the script code at dodo.py +## Regenerating .mo bundles -1. convert strings from .properties files to .mo for bundling -This step combines the .properties files into a single file, discarding any strings that are not present in code and normalizing curly quotes and unrecognized characters in the strings it keeps. (These files are separate because they are pulled from separate translation sources internally.) -> python -m doit combine_property_files +Run these when properties files change. Each step is idempotent and will overwrite the existing output. More details about implementation are in the script code at `dodo.py`. -2. Convert the combined .properties file into a .po file (these are human readable) +Or run the full pipeline in one command: +> python -m doit localize + +Individual steps: + +1. Combine .properties files into a single per-locale `combined.tmp`. Discards strings not present in code and normalizes curly quotes. +> python -m doit properties + +2. Convert each `combined.tmp` into a human-readable .po file: > python -m doit po -3. Convert the .po files into .mo files (these are not human readable) -This also checks the .mo files for validity by loading them with gettext +3. Convert the .po files into binary .mo files. Also validates each .mo by loading it with gettext: > python -m doit mo ## Optional reorganization task -Move all strings from extra.properties to the bottom of tabcmd_messages_xx.properties: +Move all strings from `extra.properties` to the bottom of `tabcmd_messages_xx.properties`: > python -m doit move_tabcmd_strings This consolidates all strings into the main tabcmd_messages files and clears the extra.properties files. \ No newline at end of file diff --git a/contributing.md b/contributing.md index 97ee80d9..c67179c4 100644 --- a/contributing.md +++ b/contributing.md @@ -138,25 +138,6 @@ The version reflected in the executable (tabcmd -v) is stored in a metadata file -### Localization - -Strings should be added/edited in /tabcmd/locales/en/{name}.properties by id and referred to in code as -> string = _("string.id") - -- regenerate updated strings for packaging as exe -> python -m doit combine_property_files po mo - - -### Versioning - -Versioning is done with setuptools_scm and based on git tags. The version number will be x.y.dev0.dirty except for commits with a new version tag. - This is pulled from the git state, and to get a clean version like "v2.1.0", you must be on a commit with the tag "v2.1.0" (Creating a Github release also creates a tag on the selected branch.) - -The version reflected in the executable (tabcmd -v) is stored in a metadata file created by a .doit script: -> python -m doit version - - - ### Packaging Packaging for release is done in a github action and should not need to be done locally. diff --git a/dodo.py b/dodo.py index 9d193c9b..5f3a6350 100644 --- a/dodo.py +++ b/dodo.py @@ -2,7 +2,6 @@ import os import subprocess import sys -import setuptools_scm LOCALES = ["en", "de", "es", "fr", "ga", "it", "pt", "sv", "ja", "ko", "zh"] @@ -103,35 +102,13 @@ def merge(): print("Combined strings for {} to {}".format(current_locale, OUTPUT_FILE)) uniquify_file(OUTPUT_FILE) - """Search loc files for each string used in code - print an error if not found. - Uses enhanced check_strings.py script for validation. - """ - - def enforce_strings_present(): - print("\n***** Verify that all string keys are present using check_strings validator") - - # English must be processed FIRST for validation baseline, others can be in any order - locales_ordered = ["en"] + [loc for loc in LOCALES if loc != "en"] - result = subprocess.run( - ["python", "bin/i18n/check_strings.py", "--mode", "build", "--locales"] + locales_ordered, - capture_output=True, - text=True, - ) - - # Print the output from the validation script - if result.stdout: - print(result.stdout) - if result.stderr: - print(result.stderr, file=sys.stderr) - - if result.returncode != 0: - print("VALIDATION FAILED: Missing localization strings found") - exit(1) - else: - print("All string validations passed") + # String-key coverage is validated by .github/workflows/check-strings.yml + # (running bin/i18n/check_strings.py directly) on every PR. Missing non-en + # keys are handled at runtime by the English fallback in localize.py, so + # they should not fail this build step. return { - "actions": [process_code, merge, enforce_strings_present], + "actions": [process_code, merge], "verbosity": 2, } @@ -382,7 +359,7 @@ def task_version(): def write_for_pyinstaller(): import pyinstaller_versionfile - import os + import setuptools_scm version = setuptools_scm.get_version(local_scheme="no-local-version") numeric_version = version.replace("dev", "") @@ -455,7 +432,7 @@ def uniquify_file(filename): ) ) else: - print("Saved {} sorted unique lines to {}".format(len(uniques), filename)) + print("Saved {} sorted unique lines to {}".format(len(unique_lines), filename)) def task_clean_all(): diff --git a/pyproject.toml b/pyproject.toml index 0456f3c7..b450b145 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ test = [ "types-requests", "types-setuptools" ] -localize = ["doit", "ftfy"] +localize = ["doit", "ftfy", "click"] package = ["doit", "pyinstaller==5.13.1", "setuptools==81"] versioning = [ "doit", diff --git a/tabcmd/execution/localize.py b/tabcmd/execution/localize.py index cc8baf1c..6547bccd 100644 --- a/tabcmd/execution/localize.py +++ b/tabcmd/execution/localize.py @@ -76,7 +76,11 @@ def define_locale_dir(logger): def _load_language(current_locale, domain, logger): locale_dir = define_locale_dir(logger) - language: gettext.NullTranslations = gettext.translation(domain, locale_dir, languages=[current_locale]) + # Chain "en" behind the preferred locale so a msgid missing from the + # preferred catalog falls back to the English msgstr per-key, instead of + # surfacing the raw msgid to the user. + languages = [current_locale] if current_locale == "en" else [current_locale, "en"] + language: gettext.NullTranslations = gettext.translation(domain, locale_dir, languages=languages) language.install() # I believe this is the expensive call _ = language.gettext return _ diff --git a/tabcmd/locales/de/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/de/LC_MESSAGES/tabcmd.mo index cac49e59..bb3aab6b 100644 Binary files a/tabcmd/locales/de/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/de/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/en/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/en/LC_MESSAGES/tabcmd.mo index a067f324..3a7c7bd5 100644 Binary files a/tabcmd/locales/en/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/en/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/es/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/es/LC_MESSAGES/tabcmd.mo index db1cd9b8..971f2d80 100644 Binary files a/tabcmd/locales/es/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/es/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/fr/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/fr/LC_MESSAGES/tabcmd.mo index 70813c35..7fb39edf 100644 Binary files a/tabcmd/locales/fr/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/fr/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/ga/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/ga/LC_MESSAGES/tabcmd.mo index ecc45de2..be01907c 100644 Binary files a/tabcmd/locales/ga/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/ga/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/it/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/it/LC_MESSAGES/tabcmd.mo index 50a2e9f7..81bf7744 100644 Binary files a/tabcmd/locales/it/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/it/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/ja/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/ja/LC_MESSAGES/tabcmd.mo index ce6b79fa..85511455 100644 Binary files a/tabcmd/locales/ja/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/ja/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/ko/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/ko/LC_MESSAGES/tabcmd.mo index 0b1ea7d1..0d307a70 100644 Binary files a/tabcmd/locales/ko/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/ko/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/pt/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/pt/LC_MESSAGES/tabcmd.mo index fa709d8a..6bfc41ef 100644 Binary files a/tabcmd/locales/pt/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/pt/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/sv/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/sv/LC_MESSAGES/tabcmd.mo index e329a6a8..0fa1d20a 100644 Binary files a/tabcmd/locales/sv/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/sv/LC_MESSAGES/tabcmd.mo differ diff --git a/tabcmd/locales/zh/LC_MESSAGES/tabcmd.mo b/tabcmd/locales/zh/LC_MESSAGES/tabcmd.mo index 3c252067..04786bd3 100644 Binary files a/tabcmd/locales/zh/LC_MESSAGES/tabcmd.mo and b/tabcmd/locales/zh/LC_MESSAGES/tabcmd.mo differ diff --git a/tests/commands/test_localize.py b/tests/commands/test_localize.py index 500cd0cc..9f7e5fdc 100644 --- a/tests/commands/test_localize.py +++ b/tests/commands/test_localize.py @@ -2,6 +2,7 @@ import locale import sys import unittest +import tabcmd.execution.localize as localize_module from tabcmd.execution.localize import set_client_locale, _get_default_locale @@ -40,6 +41,26 @@ def test_en_smoke_line_processed(self): assert translations("importcsvsummary.line.processed") == "Lines processed: {0}" + def test_fallback_to_english_when_key_missing_in_locale(self): + # session.errors.password_file_not_found exists in en but not fr. + # Loading fr should chain en behind it so the key resolves to the en msgstr. + localize_module.translate = None + translations = set_client_locale("fr") + assert translations is not None + result = translations("session.errors.password_file_not_found") + assert result == "Password file not found: '{0}'. Check the path passed to --password-file." + + def test_key_present_in_locale_uses_locale_translation(self): + # createsite.errors.site_name_already_exists is translated in fr; the fallback + # should NOT clobber the fr msgstr. + localize_module.translate = None + translations = set_client_locale("fr") + assert translations is not None + result = translations("createsite.errors.site_name_already_exists") + assert result != "createsite.errors.site_name_already_exists" + assert "'{0}'" in result # placeholder preserved + assert result != "There is already a site named '{0}'. Try a different site name." + # https://docs.python.org/3/library/locale.html # c:\dev\tabcmd\tabcmd\execution\localize.py:85: DeprecationWarning: # 'locale.getdefaultlocale' is deprecated and slated for removal in Python 3.15. Use setlocale(), getencoding() and getlocale() instead