diff --git a/.behaverc b/.behaverc new file mode 100644 index 0000000..a23bd87 --- /dev/null +++ b/.behaverc @@ -0,0 +1,27 @@ +[behave] +; We are not using the default path to leave room for unit tests ; also, apps. +paths = tests/acceptance + +# What is going on? Does this change anything for you? --format=plain works +;format = plain + +; Perhaps we want some of those? +;logging_clear_handlers = yes +;logging_filter = -suds + +; Show all print() statements even if tests pass. (bugs! bugs!) +stderr_capture = False +stdout_capture = False +log_capture = False + + +show_timings = False +show_source = False +show_multiline = True +show_skipped = False +show_snippets = False +summary = True + +[behave.formatters] +# We override the buggy "pretty" formatter +pretty = tests.acceptance.formatter:BeautifulFormatter \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 66b6b03..3f714ad 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,8 +3,8 @@ ### Version -#### Expected +### Expected -#### Actual +### Actual ### Additional Details diff --git a/.gitignore b/.gitignore index 09271d1..0dae912 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,6 @@ *.mo .#* -backup - # Python __pycache__/ .mypy_cache @@ -14,9 +12,15 @@ __pycache__/ # Local (secret) configuration /.env.local +# Backup +backup + # Databases (dumps?) test.db main.db +# Operating System +.directory + # IDE /.idea/ diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..8a3b386 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,69 @@ +# Install + +## Install using Docker + +> see root README + +--- + +## Install using virtualenv + +> It's usually less painful to install using Docker instead. + + +### Install the required packages + + +#### Ubuntu & Debian + + + sudo apt install python3 python3-dev virtualenv + sudo apt install postgresql-server-dev-all + + +### Set up a virtualenv + + virtualenv .venv --python=python3 + source .venv/bin/activate + + +### Install Python dependencies + + pip install -r requirements.txt + + +### Configure your instance + + cp .env .env.local + nano .env.local + … + source .env.local + + +### Bootstrap PostgreSQL Database + + sudo -u postgres psql + postgres=# ALTER USER postgres PASSWORD 'MyNotSoSecretPassword'; + postgres=# CREATE DATABASE mvapi; + python manage.py migrate + + +### Create an Admin user + + python manage.py createsuperuser + + +### Develop + +To run each time you're setting up your shell. + + source .venv/bin/activate + source env.local + python manage.py runserver + +Visit http://localhost:8000/admin + + +### Test + + python manage.py behave diff --git a/README.md b/README.md index 0b5f8f2..dd67d6a 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ You certainly want to apply the database migrations with: ./docker/test.sh +## Create databases migrations + + sudo ./docker/makemigrations.sh + ## Local development diff --git a/app/crud.py b/app/crud.py index ce674b5..24653a7 100644 --- a/app/crud.py +++ b/app/crud.py @@ -227,7 +227,7 @@ def _check_if_ref_exists(db: Session, ref: str): def create_election( - db: Session, election: schemas.ElectionCreate + db: Session, election: schemas.ElectionCreate, ) -> schemas.ElectionGet: # We create first the election # without candidates and grades @@ -236,7 +236,7 @@ def create_election( raise errors.InconsistentDatabaseError("Can not create election") election_ref = str(db_election.ref) - # Then, we add separatly candidates and grades + # Then, we add separately candidates and grades for candidate in election.candidates: params = candidate.model_dump() candidate = schemas.CandidateCreate(**params) diff --git a/app/database.py b/app/database.py index 3fb85eb..fbba13d 100644 --- a/app/database.py +++ b/app/database.py @@ -1,7 +1,9 @@ from __future__ import annotations + +from contextvars import ContextVar from urllib.parse import quote from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker, declarative_base +from sqlalchemy.orm import sessionmaker, declarative_base, Session from .settings import settings @@ -19,7 +21,7 @@ engine = create_engine(database_url) SessionLocal: sessionmaker = sessionmaker( # type: ignore - autocommit=False, autoflush=False, bind=engine + autocommit=False, autoflush=False, bind=engine, ) Base = declarative_base() @@ -31,3 +33,5 @@ def get_db(): yield db finally: db.close() + +db_session: ContextVar[Session] = ContextVar('db_session') diff --git a/app/main.py b/app/main.py index 61b62df..19c5c48 100644 --- a/app/main.py +++ b/app/main.py @@ -77,7 +77,7 @@ async def inconsistent_database_exception_handler( return JSONResponse( status_code=500, content={ - "message": f"A serious error has occured with {exc.name}. {exc.details or ''}" + "message": f"A serious error has occurred with {exc.name}. {exc.details or ''}" }, ) diff --git a/app/settings.py b/app/settings.py index 0c247e5..3b75172 100644 --- a/app/settings.py +++ b/app/settings.py @@ -3,7 +3,10 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", extra="ignore") + model_config = SettingsConfigDict( + env_file=".env.local", + extra="ignore", + ) sqlite: bool = False diff --git a/libs/majority_judgment.py b/libs/majority_judgment.py new file mode 100644 index 0000000..e69de29 diff --git a/mvapi/settings.py b/mvapi/settings.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements-dev.txt b/requirements-dev.txt index 3963e29..e2d5622 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,23 @@ alembic==1.8.1 types-python-jose==3.3.4 types-python-dateutil==2.8.2 mypy==1.15.0 -httpx>=0.22.0 \ No newline at end of file +httpx>=0.22.0 + +# Test Runners +#nose==1.3.7 +behave==1.2.6 +#django-nose==1.4.6 +#behave-django==1.3.0 + +# Assertion library +pyhamcrest==2.0.2 +#git+git://github.com/domi41/PyHamcrest@hack-i18n#egg=pyhamcrest + +# Test Metrics +coverage>=4.5.1 + +# Natural Language Processor for numbers +text2num>=2.2.0 + +# Markup language for humans +#PyYAML==5.3.1 diff --git a/requirements.txt b/requirements.txt index a784978..16846d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ sqlalchemy==2.0.40 pydantic==2.11.3 psycopg2==2.9.5 git+https://github.com/MieuxVoter/majority-judgment-library-python -python-jose==3.3.0 +python-jose==3.4.0 python-dateutil==2.8.2 -pydantic-settings==2.9.1 \ No newline at end of file +pydantic-settings==2.9.1 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/acceptance/README.md b/tests/acceptance/README.md new file mode 100644 index 0000000..f4fa23d --- /dev/null +++ b/tests/acceptance/README.md @@ -0,0 +1,4 @@ + +Run all scenarios except the ones tagged with `@wip`: + + python manage.py behave --no-skipped --tags=~@wip diff --git a/tests/acceptance/__init__.py b/tests/acceptance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/acceptance/environment.py b/tests/acceptance/environment.py new file mode 100644 index 0000000..1a52b6c --- /dev/null +++ b/tests/acceptance/environment.py @@ -0,0 +1,66 @@ +""" +Environment module for acceptance testing of the scenaristic constitution. +https://behave.readthedocs.io/en/latest/api.html#environment-file-functions +""" + +from behave.runner import Context +from locale import setlocale, LC_TIME + +from steps.context_main import reset_context as reset_main_context +from steps.tools_i18n import guess_locale + + +# Since we expect most of our step defs to require the flexibility of regular +# expressions (I18N, epicene) we make regular expressions the default. +# This is at the expense of automatic type casting of step variables. +from behave import use_step_matcher +use_step_matcher("re") +# You can still override this right before your step def if you want, +# by calling use_step_matcher() with one of the following: +# "parse" (factory default), "cfparse" +# and calling it again with "re" after your step def. (it uses a `global`) + + +def before_all(context: Context): + """ + Ran before the whole shooting match. + """ + pass + + +def before_feature(context, feature): + """ + Ran before _each_ feature file is exercised. + """ + context_locale = guess_locale(context) + setlocale(LC_TIME, "%s.UTF-8" % context_locale) + # REQUIRES CUSTOM FORK OF HAMCREST + # from hamcrest import set_locale as set_hamcrest_locale + # set_hamcrest_locale(context_locale) + ################################## + + +def before_scenario(context, scenario): + """ + Ran before _each_ scenario is run. + """ + reset_main_context(context) + pass + + +def after_scenario(context, scenario): + """ + Ran after _each_ scenario is run. + """ + pass + + +def after_step(context, step): + """ + Ran after _each_ step is run. + """ + pass + + +def django_ready(context): + context.django = True diff --git a/tests/acceptance/features/fr_FR b/tests/acceptance/features/fr_FR new file mode 160000 index 0000000..91486c9 --- /dev/null +++ b/tests/acceptance/features/fr_FR @@ -0,0 +1 @@ +Subproject commit 91486c99175021af6c595043b59ed56ca5f96632 diff --git a/tests/acceptance/formatter.py b/tests/acceptance/formatter.py new file mode 100644 index 0000000..9d8fd4d --- /dev/null +++ b/tests/acceptance/formatter.py @@ -0,0 +1,80 @@ +import six +from behave.formatter.pretty import PrettyFormatter +from behave.formatter.base import Formatter +from behave.textutil import indent + + +class BeautifulFormatter(PrettyFormatter): + """ + Quick patch for the most annoying bug of the PrettyFormatter. + Eventually we should move to our own CustomFormatter, + so that captured logs and stdout are printed AFTER the step, + like in high-quality gherkin runners in other languages. + """ + def result(self, step): + if not self.monochrome: + lines = self.step_lines + 1 + if self.show_multiline: + if step.table: + lines += len(step.table.rows) + 1 + if step.text: + lines += len(step.text.splitlines()) + 2 + # self.stream.write(up(lines)) # NO GOD PLEASE NO + arguments = [] + location = None + if self._match: + arguments = self._match.arguments + location = self._match.location + self.print_step(step.status, arguments, location, True) + if step.error_message: + self.stream.write(indent(step.error_message.strip(), u" ")) + self.stream.write("\n\n") + self.stream.flush() + + def match(self, match): + self._match = match + self.print_statement() + self.stream.flush() + + +class CustomFormatter(Formatter): + """ + Half-assed attempt at making our own formatter entirely. + We don't use this for now, we're using the BeautifulFormatter above. + """ + + # indentation = "\t" + indentation = " " + + def indent(self, text): + p = self.indentation + lines = text.split("\n") + return "\n".join(["%s%s" % ("" if "" == line else p, line) for line in lines]) + + def feature(self, feature): + self.stream.write(u"\n") + self.stream.write(u"%s: %s\n" % ( + feature.keyword, + feature.name, + )) + for line in feature.description: + self.stream.write(self.indent(u"%s\n" % line)) + self.stream.flush() + + def scenario(self, scenario): + self.stream.write(u"\n") + self.stream.write(u"%s: %s\n" % ( + scenario.keyword, + scenario.name, + )) + self.stream.flush() + + def result(self, step): + self.stream.write(self.indent(u"%s %s\n" % ( + step.keyword, + step.name, + ))) + if step.error_message: + self.stream.write(self.indent(step.error_message.strip())) + self.stream.write("\n") + self.stream.flush() diff --git a/tests/acceptance/steps/README.md b/tests/acceptance/steps/README.md new file mode 100644 index 0000000..1930550 --- /dev/null +++ b/tests/acceptance/steps/README.md @@ -0,0 +1,35 @@ + +# Step Definitions and Tooling + +Steps can be `esoteric` and access the inside of the application, +or `exoteric` and access the application from outside, from REST. + + +## Caveats + +### Usage of the generic `@step` def wrapper + +We are using: + +```python +from behave import step +``` + +We could use: + +```python +from behave import given, when, then +``` + +But PyCharm 2020.1 won't understand the above. +It does understand `step`. + +During the initial sprint, the IDE sugar was deemed more important. +Feel free to open an issue to discuss and|or re-evaluate this. + + +### I18N of error messages + +Hamcrest has no I18N support whatsoever: +https://github.com/hamcrest/PyHamcrest/blob/632840d9ffe7fd4e9ea9ad6ac1db9ff3871cb984/src/hamcrest/core/assert_that.py#L65 + diff --git a/tests/acceptance/steps/__init__.py b/tests/acceptance/steps/__init__.py new file mode 100644 index 0000000..913d27e --- /dev/null +++ b/tests/acceptance/steps/__init__.py @@ -0,0 +1,4 @@ + +# What can we do from here? +# Not sure whether behave's exec() allows us to hook things in here at all. +# Be warned. HERE BE DRAGONS diff --git a/tests/acceptance/steps/context_main.py b/tests/acceptance/steps/context_main.py new file mode 100644 index 0000000..f9f811c --- /dev/null +++ b/tests/acceptance/steps/context_main.py @@ -0,0 +1,17 @@ +""" +What we need: +- Share context between step defs (including from different files) +- Easy and scalable context reset on each scenario +- Multiple contexts (do we?) + +Context variables in themselves must be as scarce as possible, +and only added after careful consideration, though. + +So far we're using the PatchedContext instance provided by behave. +""" + + +def reset_context(context): + context.users = dict() + context.that_user = None + context.that_poll = None diff --git a/tests/acceptance/steps/steps_esoteric_scrutin.py b/tests/acceptance/steps/steps_esoteric_scrutin.py new file mode 100644 index 0000000..f261ee1 --- /dev/null +++ b/tests/acceptance/steps/steps_esoteric_scrutin.py @@ -0,0 +1,55 @@ +""" +Esoteric (coming from within) steps about polls. +These steps deal with the database directly, and NOT with the REST API. +""" + +from behave import given, when, then, step +from fastapi import Depends +from hamcrest import assert_that, equal_to +from datetime import datetime as clock, timedelta + +from sqlalchemy.orm import Session +# from sqlalchemy.testing import db + +from app import schemas +from app.crud import create_election, create_candidate +# from app.database import get_db, db_session +from app.models import Grade +# from app.models import Election, Candidate + +from tools_nlp import parse_amount, parse_yaml +from tools_dbal import count_polls + + +# db: Session = Depends(get_db) + + +# @given +@step(u"un(?: autre)? scrutin comme suit *:?") +def there_is_a_poll_like_so(context): + data = parse_yaml(context) + # db: Session = Depends(get_db) + poll = create_election( + db=db, + election=schemas.ElectionCreate( + name=data.get('title'), + candidates=data.get('candidates'), + grades=data.get('grades', [ + Grade(name="Excellent", description="", value=6), + Grade(name="Très Bien", description="", value=5), + Grade(name="Bien", description="", value=4), + Grade(name="Assez Bien", description="", value=3), + Grade(name="Passable", description="", value=2), + Grade(name="Insuffisant", description="", value=1), + Grade(name="À Rejeter", description="", value=0), + ]), + ), + ) + context.that_poll = poll + + +# @then +@step(u"(?:qu')?il(?: ne)? devrait(?: maintenant)? y avoir (?P.+) scrutins? dans la base de données") +def there_should_be_n_polls(context, amount): + amount = parse_amount(context, amount) + assert_that(count_polls(), equal_to(amount)) diff --git a/tests/acceptance/steps/steps_esoteric_user.py b/tests/acceptance/steps/steps_esoteric_user.py new file mode 100644 index 0000000..8dfdb38 --- /dev/null +++ b/tests/acceptance/steps/steps_esoteric_user.py @@ -0,0 +1,35 @@ +""" +Esoteric (coming from within) steps about users. +These steps deal with the database directly, and NOT with the REST API. +""" + +from behave import given, when, then, step +from hamcrest import assert_that, equal_to + +from tools_nlp import parse_amount +from tools_dbal import count_users, make_user + + +############################################################################### + +# Aliases? +# ⋅e = (?:[⋅.-]?e|) +# ⋅ne = (?:[⋅.-]?ne|) +# ⋅nes = (?:[⋅.-]?ne|)s? + +############################################################################### + + +# @given +@step(u"un(?:[⋅.-]?e|) citoyen(?:[⋅.-]?ne|) nommé(?:[⋅.-]?e|) (?P.+)") +def create_citizen_named(context, name): + print("Creating citizen named `%s'…\n" % name) + user = make_user(context, name) + context.that_user = user + + +# @then +@step(u"(?:qu')?il(?: ne)? devrait y avoir (?P.+) citoyen(?:[⋅.-]?ne?|)s? dans la base de données") +def there_should_be_n_users(context, amount): + amount = parse_amount(context, amount) + assert_that(count_users(context), equal_to(amount)) diff --git a/tests/acceptance/steps/steps_ouputting.py b/tests/acceptance/steps/steps_ouputting.py new file mode 100644 index 0000000..c241508 --- /dev/null +++ b/tests/acceptance/steps/steps_ouputting.py @@ -0,0 +1,14 @@ +""" +These steps should [ae]ffect nothing. +They help keep the features idiomatic, engaging, etc. +""" + +from behave import step +from tools_dbal import find_user + + +@step(u"(?:j(?:e |'))?(?:débogue|affiche)(?: l[ea])? citoyen(?:[⋅.-]?ne|)(?: nommé(?:[⋅.-]?e|))? (?P.+)") +@step(u"(?:I )?print(?: the)? user(?: named)? (?P.+)") +def print_user(context, name): + user = find_user(name) + print("%s\n" % user) diff --git a/tests/acceptance/steps/steps_poetry.py b/tests/acceptance/steps/steps_poetry.py new file mode 100644 index 0000000..6b87c51 --- /dev/null +++ b/tests/acceptance/steps/steps_poetry.py @@ -0,0 +1,16 @@ +""" +These steps should [ae]ffect nothing. +They help keep the features idiomatic, engaging, etc. +""" + +from behave import step + + +@step(u"(?:etc[.]?|…|[.]{3,}|[?!]+)[?!]*") +def et_caetera(context): + pass # nothing is cool + + +@step(u"ce n'est pas tout *!*:?") +def wait_there_is_more(context): + pass # nothing is cool diff --git a/tests/acceptance/steps/steps_rest_scrutin.py b/tests/acceptance/steps/steps_rest_scrutin.py new file mode 100644 index 0000000..fdaa611 --- /dev/null +++ b/tests/acceptance/steps/steps_rest_scrutin.py @@ -0,0 +1,58 @@ +""" +REST steps about polls. +These steps deal with the REST API, not the database. +""" + +from behave import given, when, then, step +from hamcrest import assert_that, equal_to + +from toolbox import parse_actor, parse_yaml, parse_grades, find_poll, fail + + +############################################################################### + + +# @when +@step(u"(?P.+) crée un(?: autre)? scrutin comme suit *:?") +def actor_creates_a_poll_like_so(context, actor): + data = parse_yaml(context) + actor = parse_actor(context, actor) + title = data.get('title') + actor.post('/polls', data={ + 'title': title, + 'candidates': data.get('candidates'), + 'num_grades': data.get('grades', 7), + }) + created_poll = find_poll(title, relax=True) + if created_poll: + context.that_poll = created_poll + + +# @when +@step(u"(?P.+) (?:vote|juge les candidats)(?: comme suit)? (?:sur|de) ce scrutin(?: comme suit)? *:?") +def actor_judges_candidates_of_that_poll_like_so(context, actor): + actor = parse_actor(context, actor) + data = parse_yaml(context) + poll = context.that_poll + grades = parse_grades(context, data, poll) + actor.post("/judgments", data={ + 'election': poll.id, + 'grades_by_candidate': grades, + }) + + +# @when +@step(u"l[ea] vainqueur(?:[⋅.-]?e)? de ce scrutin devrait être: (?P.+)") +def winner_of_that_poll_should_be(context, candidate): + actor = parse_actor(context, "C0h4N") + poll = context.that_poll + # "/results/{poll.id}/" + response = actor.get(f"/polls/{poll.id}/results") + + data = response.json() + # data example : + # [{'name': 'Islande', 'id': 1, 'grade': 5, + # 'profile': {'0': 0, '1': 0, '2': 0, '3': 0, '4': 0, '5': 1, '6': 1}}, + # {'name': 'France', 'id': 0, 'grade': 1, + # 'profile': {'0': 0, '1': 1, '2': 1, '3': 0, '4': 0, '5': 0, '6': 0}}] + assert_that(data[0]['name'], equal_to(candidate)) diff --git a/tests/acceptance/steps/steps_time.py b/tests/acceptance/steps/steps_time.py new file mode 100644 index 0000000..2880354 --- /dev/null +++ b/tests/acceptance/steps/steps_time.py @@ -0,0 +1,15 @@ +""" +Basic control of time. +""" + +from behave import step +from time import sleep + + +from tools_nlp import parse_amount + + +@step(u"j'attends (?P.+) secondes") +def i_wait_for_seconds(context, amount): + amount = parse_amount(context, amount) + sleep(amount) diff --git a/tests/acceptance/steps/toolbox.py b/tests/acceptance/steps/toolbox.py new file mode 100644 index 0000000..b13ddf7 --- /dev/null +++ b/tests/acceptance/steps/toolbox.py @@ -0,0 +1,67 @@ +""" +The intent is for toolbox to be a sort of shortcut, providing most tools. +Not sure this is the python way. +""" + +# from httpx import Client +from fastapi.testclient import TestClient as Client + +from app.main import app +# Keep these, it is used by those who import toolbox +from tools_dbal import * +from tools_nlp import * + + +class Actor(object): + """ + Light wrapper around HTTP clients, where we can put our nitty-gritty. + """ + + def __init__(self, name=None) -> None: + super().__init__() + self.name = name + self.client = Client(app) + self.last_response = None + + def adjust_path(self, path): + if path.startswith('/'): + path = path[1:] + return "/api/election/%s" % path + + def handle_possible_failure(self, method, path, response): + # FIXME: I18N + if response.status_code >= 400: + print("%s %s (%d)\n" % (method, path, response.status_code)) + print(response.content) + raise AssertionError("Request should succeed.") + return response + + def get(self, path, data=None, safe_to_fail=False): + path = self.adjust_path(path) + response = self.client.get(path=path, data=data) + self.last_response = response + + if not safe_to_fail: + self.handle_possible_failure('GET', path, response) + return response + + def post(self, path, data, safe_to_fail=False): + path = self.adjust_path(path) + response = self.client.post(path=path, data=data) + self.last_response = response + + if not safe_to_fail: + self.handle_possible_failure('POST', path, response) + return response + + +def parse_actor(context, actor_name): + if 'actors' not in context: + context.actors = dict() + if actor_name not in context.actors: + context.actors[actor_name] = Actor(name=actor_name) + return context.actors[actor_name] + + +def fail(message): # TBD; How to I18N AssertionError? + raise AssertionError(message) \ No newline at end of file diff --git a/tests/acceptance/steps/tools_dbal.py b/tests/acceptance/steps/tools_dbal.py new file mode 100644 index 0000000..7c7adcf --- /dev/null +++ b/tests/acceptance/steps/tools_dbal.py @@ -0,0 +1,53 @@ +""" +Local database abstraction layer for step defs. +""" +from behave.runner import Context +from fastapi import Depends +from sqlalchemy.orm import Session +# from sqlalchemy.testing import db + +from app import errors +from app.crud import get_election +from app.database import get_db +from app.models import Election + + +db: Session = Depends(get_db) + + +class User: + def __init__(self, username): + self.username = username + + +def make_user(context: Context, username): + user = User(username) + context.users[username] = user + return user + + +def count_users(context: Context): + return len(context.users.items()) + + +def find_user(context: Context, username: str, relax=False): + user = context.users.get(username, None) + if user is not None: + return user + if not relax: + raise ValueError("No user found matching `%s`." % username) + return None + + +def count_polls(): + return db.query(Election).count() + + +def find_poll(identifier, relax=False): + try: + return get_election(db, identifier) + except errors.NotFoundError: + if not relax: + raise ValueError("No poll found matching `%s`." % identifier) + return None + diff --git a/tests/acceptance/steps/tools_i18n.py b/tests/acceptance/steps/tools_i18n.py new file mode 100644 index 0000000..a772a5f --- /dev/null +++ b/tests/acceptance/steps/tools_i18n.py @@ -0,0 +1,218 @@ +""" +Internationalization tools for step definitions. +""" + +languages = [ + # ISO 639-1, Full Name + ('aa', 'Afar'), + ('ab', 'Abkhazian'), + ('af', 'Afrikaans'), + ('ak', 'Akan'), + ('sq', 'Albanian'), + ('am', 'Amharic'), + ('ar', 'Arabic'), + ('an', 'Aragonese'), + ('hy', 'Armenian'), + ('as', 'Assamese'), + ('av', 'Avaric'), + ('ae', 'Avestan'), + ('ay', 'Aymara'), + ('az', 'Azerbaijani'), + ('ba', 'Bashkir'), + ('bm', 'Bambara'), + ('eu', 'Basque'), + ('be', 'Belarusian'), + ('bn', 'Bengali'), + ('bh', 'Bihari languages'), + ('bi', 'Bislama'), + ('bo', 'Tibetan'), + ('bs', 'Bosnian'), + ('br', 'Breton'), + ('bg', 'Bulgarian'), + ('my', 'Burmese'), + ('ca', 'Catalan; Valencian'), + ('cs', 'Czech'), + ('ch', 'Chamorro'), + ('ce', 'Chechen'), + ('zh', 'Chinese'), + ('cu', 'Church Slavic'), + ('cv', 'Chuvash'), + ('kw', 'Cornish'), + ('co', 'Corsican'), + ('cr', 'Cree'), + ('cy', 'Welsh'), + ('cs', 'Czech'), + ('da', 'Danish'), + ('de', 'German'), + ('dv', 'Divehi; Dhivehi; Maldivian'), + ('nl', 'Dutch; Flemish'), + ('dz', 'Dzongkha'), + ('el', 'Greek'), + ('en', 'English'), + ('eo', 'Esperanto'), # <3 + ('et', 'Estonian'), + ('eu', 'Basque'), + ('ee', 'Ewe'), + ('fo', 'Faroese'), + ('fa', 'Persian'), + ('fj', 'Fijian'), + ('fi', 'Finnish'), + ('fr', 'French'), + ('fy', 'Western Frisian'), + ('ff', 'Fulah'), + ('ga', 'Georgian'), + ('de', 'German'), + ('gd', 'Gaelic; Scottish Gaelic'), + ('ga', 'Irish'), + ('gl', 'Galician'), + ('gv', 'Manx'), + ('gn', 'Guarani'), + ('gu', 'Gujarati'), + ('ht', 'Haitian; Haitian Creole'), + ('ha', 'Hausa'), + ('he', 'Hebrew'), + ('hz', 'Herero'), + ('hi', 'Hindi'), + ('ho', 'Hiri Motu'), + ('hr', 'Croatian'), + ('hu', 'Hungarian'), + ('hy', 'Armenian'), + ('ig', 'Igbo'), + ('is', 'Icelandic'), + ('io', 'Ido'), + ('ii', 'Sichuan Yi; Nuosu'), + ('iu', 'Inuktitut'), + ('ie', 'Interlingue; Occidental'), + ('ia', 'Interlingua'), # !? + ('id', 'Indonesian'), + ('ik', 'Inupiaq'), + ('is', 'Icelandic'), + ('it', 'Italian'), + ('jv', 'Javanese'), + ('ja', 'Japanese'), + ('kl', 'Kalaallisut; Greenlandic'), + ('kn', 'Kannada'), + ('ks', 'Kashmiri'), + ('ka', 'Georgian'), + ('kr', 'Kanuri'), + ('kk', 'Kazakh'), + ('km', 'Central Khmer'), + ('ki', 'Kikuyu; Gikuyu'), + ('rw', 'Kinyarwanda'), + ('ky', 'Kirghiz; Kyrgyz'), + ('kv', 'Komi'), + ('kg', 'Kongo'), + ('ko', 'Korean'), + ('kj', 'Kuanyama; Kwanyama'), + ('ku', 'Kurdish'), + ('lo', 'Lao'), + ('la', 'Latin'), + ('lv', 'Latvian'), + ('li', 'Limburgan; Limburger; Limburgish'), + ('ln', 'Lingala'), + ('lt', 'Lithuanian'), + ('lb', 'Luxembourgish; Letzeburgesch'), + ('lu', 'Luba-Katanga'), + ('lg', 'Ganda'), + ('ma', 'Marain'), # _[⋅.-]_ + ('mk', 'Macedonian'), + ('mh', 'Marshallese'), + ('ml', 'Malayalam'), + ('mi', 'Maori'), + ('mr', 'Marathi'), + ('ms', 'Malay'), + ('Mi', 'Micmac'), + ('mg', 'Malagasy'), + ('mt', 'Maltese'), + ('mn', 'Mongolian'), + ('mi', 'Maori'), + ('ms', 'Malay'), + ('my', 'Burmese'), + ('na', 'Nauru'), + ('nv', 'Navajo; Navaho'), + ('nr', 'South Ndebele'), + ('nd', 'North Ndebele'), + ('ng', 'Ndonga'), + ('ne', 'Nepali'), + ('nl', 'Dutch; Flemish'), + ('nn', 'Norwegian Nynorsk; Nynorsk, Norwegian'), + ('nb', 'Bokmål, Norwegian; Norwegian Bokmål'), + ('no', 'Norwegian'), + ('oc', 'Occitan (post 1500)'), + ('oj', 'Ojibwa'), + ('or', 'Oriya'), + ('om', 'Oromo'), + ('os', 'Ossetian; Ossetic'), + ('pa', 'Panjabi; Punjabi'), + ('fa', 'Persian'), + ('pi', 'Pali'), + ('pl', 'Polish'), + ('pt', 'Portuguese'), + ('ps', 'Pushto; Pashto'), + ('qu', 'Quechua'), + ('rm', 'Romansh'), + ('ro', 'Romanian; Moldavian; Moldovan'), + ('ro', 'Romanian; Moldavian; Moldovan'), + ('rn', 'Rundi'), + ('ru', 'Russian'), + ('sg', 'Sango'), + ('sa', 'Sanskrit'), + ('si', 'Sinhala; Sinhalese'), + ('sk', 'Slovak'), + ('sk', 'Slovak'), + ('sl', 'Slovenian'), + ('se', 'Northern Sami'), + ('sm', 'Samoan'), + ('sn', 'Shona'), + ('sd', 'Sindhi'), + ('so', 'Somali'), + ('st', 'Sotho, Southern'), + ('es', 'Spanish; Castilian'), + ('sq', 'Albanian'), + ('sc', 'Sardinian'), + ('sr', 'Serbian'), + ('ss', 'Swati'), + ('su', 'Sundanese'), + ('sw', 'Swahili'), + ('sv', 'Swedish'), + ('ty', 'Tahitian'), + ('ta', 'Tamil'), + ('tt', 'Tatar'), + ('te', 'Telugu'), + ('tg', 'Tajik'), + ('tl', 'Tagalog'), + ('th', 'Thai'), + ('bo', 'Tibetan'), + ('ti', 'Tigrinya'), + ('to', 'Tonga (Tonga Islands)'), + ('tn', 'Tswana'), + ('ts', 'Tsonga'), + ('tk', 'Turkmen'), + ('tr', 'Turkish'), + ('tw', 'Twi'), + ('ug', 'Uighur; Uyghur'), + ('uk', 'Ukrainian'), + ('ur', 'Urdu'), + ('uz', 'Uzbek'), + ('ve', 'Venda'), + ('vi', 'Vietnamese'), + ('vo', 'Volapük'), + ('cy', 'Welsh'), + ('wa', 'Walloon'), + ('wo', 'Wolof'), + ('xh', 'Xhosa'), + ('yi', 'Yiddish'), + ('yo', 'Yoruba'), + ('za', 'Zhuang; Chuang'), + ('zh', 'Chinese'), + ('zu', 'Zulu') +] + + +def guess_locale(context): # naive, inefficient + for language in languages: + for tag in context.tags: + if language[0] == tag[0:2]: + return tag # eg: "fr_FR" + raise Exception("No language found. Use a tag @en_US or @fr_FR?") + diff --git a/tests/acceptance/steps/tools_nlp.py b/tests/acceptance/steps/tools_nlp.py new file mode 100644 index 0000000..1e6e469 --- /dev/null +++ b/tests/acceptance/steps/tools_nlp.py @@ -0,0 +1,93 @@ +""" +Natural Language Processing tools +""" + +import re + +from behave.runner import Context +from text_to_num import text2num + +from tools_i18n import guess_locale + + +def parse_amount(context: Context, amount_string): + """ + - Multilingual (hopefully) EN - FR - … + - Tailored for Gherkin features and behave + :param context: Context object from behave step def + :param string amount_string: + :return int|float: + """ + language = guess_locale(context)[0:2] + if 'fr' == language: + if re.match("^aucun(?:[⋅.-]?e)?$", amount_string): + return 0 + elif 'en' == language: + if re.match("^no(?:ne)?$", amount_string): + return 0 + + # return text2num(text=amount_string, lang=language, relaxed=True) + return text2num(text=amount_string, lang=language) + + +def parse_yaml(context, with_i18n=True): + """ + :param context: Context object from behave step def + :param bool with_i18n: + :return: The step pystring contents as dictionary from YAML + """ + from yaml import safe_load + data = safe_load(context.text) + + if with_i18n: + language = guess_locale(context) + # Eventually, load these maps from files, perhaps + yaml_keys_map = { + 'fr_FR': { + 'titre': 'title', + 'candidats': 'candidates', + 'candidates': 'candidates', + 'candidat⋅es': 'candidates', + 'durée': 'duration', + }, + } + # Naive mapping, not collision-resilient, but it's ok for now + if language in yaml_keys_map: + for key in [k for k in data.keys()]: + if key in yaml_keys_map[language]: + data[yaml_keys_map[language][key]] = data[key] + + return data + + +def parse_grades(context, data, poll): + grades = [] + + for candidate in poll.candidates: + if candidate not in data: + raise AssertionError("Candidate `%s' not found in `%s'." % (candidate, data)) + grade_text = data[candidate] + + grades_database = { # TBD + 'fr_FR': { + 'quality': { + '7': [ + ['à rejeter'], + ['insuffisant', 'insuffisante', 'insuffisant⋅e', 'insuffisant-e', 'insuffisant.e'], + ['passable'], + ['assez bien'], + ['bien'], + ['très bien'], + ['excellent', 'excellente', 'excellent⋅e', 'excellent-e', 'excellent.e'], + ], + }, + } + } + locale = guess_locale(context) + grades_in_order = grades_database[locale]['quality']['7'] + for k, matching_grades in enumerate(grades_in_order): + if grade_text in matching_grades: + grades.append(k) + break + + return grades