diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02bbe150e5..cdbd3cf757 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,13 @@ exclude: | (?x) # NOT INSTALLABLE ADDONS + ^base_import_async/| + ^queue_job_batch/| + ^queue_job_cron/| + ^queue_job_cron_jobrunner/| + ^queue_job_subscribe/| + ^test_queue_job/| + ^test_queue_job_batch/| # END NOT INSTALLABLE ADDONS # Files and folders generated by bots, to avoid loops ^setup/|/static/description/index\.html$| diff --git a/queue_job/README.rst b/queue_job/README.rst index f9a974d192..cdcda137ae 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -11,7 +11,7 @@ Job Queue !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:aee74431fafa74d2abc078c3686e4d9c6390bacc001cac7c4afa01b9db77ca06 + !! source digest: sha256:07592ea84e63ed35de86f099ccb855d656184ecd4a570168b8eed73aab133157 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Mature-brightgreen.png @@ -21,13 +21,13 @@ Job Queue :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github - :target: https://github.com/OCA/queue/tree/19.0/queue_job + :target: https://github.com/OCA/queue/tree/20.0/queue_job :alt: OCA/queue .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/queue-19-0/queue-19-0-queue_job + :target: https://translation.odoo-community.org/projects/queue-20-0/queue-20-0-queue_job :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png - :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=19.0 + :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=20.0 :alt: Try me on Runboat |badge1| |badge2| |badge3| |badge4| |badge5| @@ -685,7 +685,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -747,6 +747,6 @@ Current `maintainers `__: |maintainer-guewen| |maintainer-sbidoul| -This module is part of the `OCA/queue `_ project on GitHub. +This module is part of the `OCA/queue `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/queue_job/__manifest__.py b/queue_job/__manifest__.py index 03ea411f2c..c890beaf9c 100644 --- a/queue_job/__manifest__.py +++ b/queue_job/__manifest__.py @@ -2,7 +2,7 @@ { "name": "Job Queue", - "version": "19.0.2.1.3", + "version": "20.0.1.0.0", "author": "Camptocamp,ACSONE SA/NV,Odoo Community Association (OCA)", "website": "https://github.com/OCA/queue", "license": "LGPL-3", @@ -11,7 +11,7 @@ "external_dependencies": {"python": ["requests", "openupgradelib"]}, "data": [ "security/security.xml", - "security/ir.model.access.csv", + "security/ir.access.csv", "views/queue_job_views.xml", "views/queue_job_channel_views.xml", "views/queue_job_function_views.xml", @@ -27,7 +27,7 @@ "queue_job/static/src/views/**/*", ], }, - "installable": False, + "installable": True, "development_status": "Mature", "maintainers": ["guewen", "sbidoul"], "post_init_hook": "post_init_hook", diff --git a/queue_job/controllers/main.py b/queue_job/controllers/main.py index 92ff000dc8..ef0fa0baef 100644 --- a/queue_job/controllers/main.py +++ b/queue_job/controllers/main.py @@ -13,7 +13,7 @@ from werkzeug.exceptions import BadRequest, Forbidden from odoo import SUPERUSER_ID, api, http -from odoo.service.model import PG_CONCURRENCY_ERRORS_TO_RETRY +from odoo.sql_db import PG_CONCURRENCY_EXCEPTIONS_TO_RETRY from odoo.tools import config from ..delay import chain, group @@ -98,10 +98,14 @@ def _try_perform_job(cls, env, job): # TODO refactor, the relation between env and job.env is not clear assert env.cr is job.env.cr with _prevent_commit(env.cr): - job.perform() - # Triggers any stored computed fields before calling 'set_done' - # so that will be part of the 'exec_time' - env.flush_all() + # On failure, rolling back the savepoint releases the row locks + # the job took on the records it updated: the failure is handled + # from another cursor, which may write on the same records (e.g. + # the on fail hook). + # On success, leaving the savepoint flushes, before 'set_done', so + # that the stored computed fields are part of the 'exec_time'. + with env.cr.savepoint(): + job.perform() job.set_done() job.store() env.flush_all() @@ -123,7 +127,7 @@ def _enqueue_dependent_jobs(cls, env, job): except OperationalError as err: # Automatically retry the typical transaction serialization # errors - if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: + if not isinstance(err, PG_CONCURRENCY_EXCEPTIONS_TO_RETRY): raise if tries >= DEPENDS_MAX_TRIES_ON_CONCURRENCY_FAILURE: _logger.error( @@ -148,7 +152,7 @@ def _enqueue_dependent_jobs(cls, env, job): @classmethod def _runjob(cls, env: api.Environment, job: Job) -> None: def retry_postpone(job, message, seconds=None): - job.env.clear() + job.env.transaction.clear() with job.in_temporary_env(): job.postpone(result=message, seconds=seconds) job.set_pending(reset_retry=False) @@ -160,7 +164,7 @@ def retry_postpone(job, message, seconds=None): except OperationalError as err: # Automatically retry the typical transaction serialization # errors - if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: + if not isinstance(err, PG_CONCURRENCY_EXCEPTIONS_TO_RETRY): raise _logger.debug("%s OperationalError, postponed", job) @@ -181,7 +185,7 @@ def retry_postpone(job, message, seconds=None): traceback.print_exc(file=buff) traceback_txt = buff.getvalue() _logger.error(traceback_txt) - job.env.clear() + job.env.transaction.clear() with job.in_temporary_env(): vals = cls._get_failure_values(job, traceback_txt, orig_exception) job.set_failed(**vals) @@ -388,6 +392,5 @@ def _create_graph_test_jobs( root_delayable.delay() - return ( - f"graph uuid: {list(root_delayable._head())[0]._generated_job.graph_uuid}" - ) + head = next(iter(root_delayable._head())) + return f"graph uuid: {head._generated_job.graph_uuid}" diff --git a/queue_job/data/queue_data.xml b/queue_job/data/queue_data.xml index 1fb70f106c..df010767b3 100644 --- a/queue_job/data/queue_data.xml +++ b/queue_job/data/queue_data.xml @@ -1,4 +1,4 @@ - + @@ -7,19 +7,9 @@ queue.job - - AutoVacuum Job Queue - - - - 1 - days - code - model.autovacuum() - - + root diff --git a/queue_job/data/queue_job_function_data.xml b/queue_job/data/queue_job_function_data.xml index 0105dbc508..31ec7ff964 100644 --- a/queue_job/data/queue_job_function_data.xml +++ b/queue_job/data/queue_job_function_data.xml @@ -1,3 +1,4 @@ + diff --git a/queue_job/delay.py b/queue_job/delay.py index 60e98dc9d6..8abc3133bc 100644 --- a/queue_job/delay.py +++ b/queue_job/delay.py @@ -58,7 +58,7 @@ class Graph: instances, although ultimately it is used for this purpose. """ - __slots__ = "_graph" + __slots__ = ("_graph",) def __init__(self, graph=None): if graph: @@ -314,7 +314,7 @@ class DelayableChain: delayable/chain/group object of the graph. """ - __slots__ = ("_graph", "__head", "__tail") + __slots__ = ("__head", "__tail", "_graph") def __init__(self, *delayables): self._graph = DelayableGraph() @@ -371,7 +371,7 @@ class DelayableGroup: delayable/chain/group object of the graph. """ - __slots__ = ("_graph", "_delayables") + __slots__ = ("_delayables", "_graph") def __init__(self, *delayables): self._graph = DelayableGraph() diff --git a/queue_job/fields.py b/queue_job/fields.py index d9f371605f..e2fcbdfe4a 100644 --- a/queue_job/fields.py +++ b/queue_job/fields.py @@ -30,7 +30,7 @@ class JobSerialized(fields.Json): _base_type = None # these are the default values when we convert an empty value - _default_json_mapping = { + _default_json_mapping = { # noqa: RUF012 dict: "{}", list: "[]", tuple: "[]", @@ -42,8 +42,8 @@ class JobSerialized(fields.Json): def __init__(self, string=SENTINEL, base_type=SENTINEL, **kwargs): super().__init__(string=string, _base_type=base_type, **kwargs) - def _setup_attrs(self, model, name): # pylint: disable=missing-return - super()._setup_attrs(model, name) + def _setup_attrs__(self, model_class, name): # pylint: disable=missing-return + super()._setup_attrs__(model_class, name) if self._base_type not in self._default_json_mapping: msg = f"{self._base_type} is not a supported base type" raise ValueError(msg) diff --git a/queue_job/i18n/de.po b/queue_job/i18n/de.po index 4db227f835..49a98b7a9f 100644 --- a/queue_job/i18n/de.po +++ b/queue_job/i18n/de.po @@ -78,11 +78,6 @@ msgstr "Argumente" msgid "Attachment Count" msgstr "Anzahl der Anhänge" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "AutoVacuum für Job-Warteschlange" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -117,6 +112,12 @@ msgstr "" msgid "Cancelled" msgstr "" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -231,6 +232,16 @@ msgstr "" msgid "Dependency Graph" msgstr "" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -469,12 +480,22 @@ msgstr "Job ist fehlgeschlagen" msgid "Jobs" msgstr "Jobs" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" @@ -945,6 +966,9 @@ msgstr "" msgid "id" msgstr "" +#~ msgid "AutoVacuum Job Queue" +#~ msgstr "AutoVacuum für Job-Warteschlange" + #~ msgid "Group By" #~ msgstr "Gruppieren nach" diff --git a/queue_job/i18n/es.po b/queue_job/i18n/es.po index d38e55959c..a2ff33074f 100644 --- a/queue_job/i18n/es.po +++ b/queue_job/i18n/es.po @@ -81,11 +81,6 @@ msgstr "Argumentos" msgid "Attachment Count" msgstr "Nº de archivos adjuntos" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "Vaciado automático de la cola de trabajos" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -120,6 +115,12 @@ msgstr "Cancelar trabajos" msgid "Cancelled" msgstr "Cancelada" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -234,6 +235,16 @@ msgstr "Dependencias" msgid "Dependency Graph" msgstr "Gráfico de dependencias" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -470,12 +481,22 @@ msgstr "Trabajo fallido" msgid "Jobs" msgstr "Trabajos" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "Trabajos para gráfico %s" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" @@ -957,6 +978,9 @@ msgstr "Pid del trabajador" msgid "id" msgstr "" +#~ msgid "AutoVacuum Job Queue" +#~ msgstr "Vaciado automático de la cola de trabajos" + #~ msgid "Group By" #~ msgstr "Agrupar por" diff --git a/queue_job/i18n/fa.po b/queue_job/i18n/fa.po index 293c8c6cc6..707e59ad93 100644 --- a/queue_job/i18n/fa.po +++ b/queue_job/i18n/fa.po @@ -78,11 +78,6 @@ msgstr "" msgid "Attachment Count" msgstr "" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -117,6 +112,12 @@ msgstr "" msgid "Cancelled" msgstr "" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -231,6 +232,16 @@ msgstr "" msgid "Dependency Graph" msgstr "" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -466,12 +477,22 @@ msgstr "" msgid "Jobs" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" diff --git a/queue_job/i18n/it.po b/queue_job/i18n/it.po index 6c43aa8dd4..9ed2b17ac8 100644 --- a/queue_job/i18n/it.po +++ b/queue_job/i18n/it.po @@ -85,11 +85,6 @@ msgstr "Argomenti" msgid "Attachment Count" msgstr "Conteggio allegati" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "Auto pulizia coda lavoro" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -124,6 +119,12 @@ msgstr "Annulla lavori" msgid "Cancelled" msgstr "Annullata" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -238,6 +239,16 @@ msgstr "Dipendenze" msgid "Dependency Graph" msgstr "Grafico dipendenza" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -473,12 +484,22 @@ msgstr "Lavro fallito" msgid "Jobs" msgstr "Lavori" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "Lavori per grafico %s" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" @@ -967,6 +988,9 @@ msgstr "PID worker" msgid "id" msgstr "id" +#~ msgid "AutoVacuum Job Queue" +#~ msgstr "Auto pulizia coda lavoro" + #~ msgid "Cancelled by {}" #~ msgstr "Annullato da {}" diff --git a/queue_job/i18n/nl.po b/queue_job/i18n/nl.po index 4e46d6b2d9..e57bc001bd 100644 --- a/queue_job/i18n/nl.po +++ b/queue_job/i18n/nl.po @@ -76,11 +76,6 @@ msgstr "" msgid "Attachment Count" msgstr "" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -115,6 +110,12 @@ msgstr "" msgid "Cancelled" msgstr "" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -229,6 +230,16 @@ msgstr "" msgid "Dependency Graph" msgstr "" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -464,12 +475,22 @@ msgstr "" msgid "Jobs" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" diff --git a/queue_job/i18n/nl_NL.po b/queue_job/i18n/nl_NL.po index 3f17ddecaf..80aeaaf07a 100644 --- a/queue_job/i18n/nl_NL.po +++ b/queue_job/i18n/nl_NL.po @@ -78,11 +78,6 @@ msgstr "Args" msgid "Attachment Count" msgstr "Aantal bijlagen" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "AutoVacuum wachtrijtaken" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -117,6 +112,12 @@ msgstr "Taken annuleren" msgid "Cancelled" msgstr "Geannuleerd" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -231,6 +232,16 @@ msgstr "Afhankelijkheden" msgid "Dependency Graph" msgstr "Afhankelijkheidsgrafiek" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -466,12 +477,22 @@ msgstr "Taak mislukt" msgid "Jobs" msgstr "Taken" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "Taken voor grafiek %s" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" @@ -954,6 +975,9 @@ msgstr "Werknemer PID" msgid "id" msgstr "" +#~ msgid "AutoVacuum Job Queue" +#~ msgstr "AutoVacuum wachtrijtaken" + #~ msgid "Cancelled by {}" #~ msgstr "Geannuleerd door {}" diff --git a/queue_job/i18n/queue_job.pot b/queue_job/i18n/queue_job.pot index d2f24262cf..6025093621 100644 --- a/queue_job/i18n/queue_job.pot +++ b/queue_job/i18n/queue_job.pot @@ -74,11 +74,6 @@ msgstr "" msgid "Attachment Count" msgstr "" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -113,6 +108,12 @@ msgstr "" msgid "Cancelled" msgstr "" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -227,6 +228,16 @@ msgstr "" msgid "Dependency Graph" msgstr "" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -462,12 +473,22 @@ msgstr "" msgid "Jobs" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" diff --git a/queue_job/i18n/tr.po b/queue_job/i18n/tr.po index badccdef35..479a7540e8 100644 --- a/queue_job/i18n/tr.po +++ b/queue_job/i18n/tr.po @@ -81,11 +81,6 @@ msgstr "Argümanlar" msgid "Attachment Count" msgstr "Ek Sayısı" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "Otomatik Temizleme İş Kuyruğu" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -120,6 +115,12 @@ msgstr "İşleri iptal et" msgid "Cancelled" msgstr "İptal Edildi" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -234,6 +235,16 @@ msgstr "Bağımlılıklar" msgid "Dependency Graph" msgstr "Bağımlılık Grafiği" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -469,12 +480,22 @@ msgstr "İş başarısız oldu" msgid "Jobs" msgstr "İşler" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "Grafik için işler %s" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" @@ -954,6 +975,9 @@ msgstr "Çalışan Pid" msgid "id" msgstr "" +#~ msgid "AutoVacuum Job Queue" +#~ msgstr "Otomatik Temizleme İş Kuyruğu" + #~ msgid "Cancelled by {}" #~ msgstr "İptal eden: {}" diff --git a/queue_job/i18n/tr_TR.po b/queue_job/i18n/tr_TR.po index adb2b9a1af..4606d0e471 100644 --- a/queue_job/i18n/tr_TR.po +++ b/queue_job/i18n/tr_TR.po @@ -76,11 +76,6 @@ msgstr "" msgid "Attachment Count" msgstr "" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -115,6 +110,12 @@ msgstr "" msgid "Cancelled" msgstr "" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -229,6 +230,16 @@ msgstr "" msgid "Dependency Graph" msgstr "" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -464,12 +475,22 @@ msgstr "" msgid "Jobs" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" diff --git a/queue_job/i18n/zh_CN.po b/queue_job/i18n/zh_CN.po index cde77ce2c9..866c24bcc5 100644 --- a/queue_job/i18n/zh_CN.po +++ b/queue_job/i18n/zh_CN.po @@ -81,11 +81,6 @@ msgstr "位置参数" msgid "Attachment Count" msgstr "附件数量" -#. module: queue_job -#: model:ir.actions.server,name:queue_job.ir_cron_autovacuum_queue_jobs_ir_actions_server -msgid "AutoVacuum Job Queue" -msgstr "自动清空作业队列" - #. module: queue_job #: model:ir.model,name:queue_job.model_base msgid "Base" @@ -120,6 +115,12 @@ msgstr "取消作业" msgid "Cancelled" msgstr "已取消" +#. module: queue_job +#. odoo-python +#: code:addons/queue_job/job.py:0 +msgid "Cancelled because a job it depends on was cancelled" +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 @@ -234,6 +235,16 @@ msgstr "依赖项" msgid "Dependency Graph" msgstr "依赖关系图" +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependency_ids +msgid "Dependency Jobs" +msgstr "" + +#. module: queue_job +#: model:ir.model.fields,field_description:queue_job.field_queue_job__dependent_ids +msgid "Dependent Jobs" +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__name msgid "Description" @@ -469,12 +480,22 @@ msgstr "作业失败" msgid "Jobs" msgstr "作业" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependent_ids +msgid "Jobs depending on this job." +msgstr "" + #. module: queue_job #. odoo-python #: code:addons/queue_job/models/queue_job.py:0 msgid "Jobs for graph %s" msgstr "图表 %s 的作业" +#. module: queue_job +#: model:ir.model.fields,help:queue_job.field_queue_job__dependency_ids +msgid "Jobs this job depends on." +msgstr "" + #. module: queue_job #: model:ir.model.fields,field_description:queue_job.field_queue_job__kwargs msgid "Kwargs" @@ -948,6 +969,9 @@ msgstr "工作进程PID" msgid "id" msgstr "" +#~ msgid "AutoVacuum Job Queue" +#~ msgstr "自动清空作业队列" + #~ msgid "Group By" #~ msgstr "分组" diff --git a/queue_job/job.py b/queue_job/job.py index 04081976c5..d987d1e6bc 100644 --- a/queue_job/job.py +++ b/queue_job/job.py @@ -461,7 +461,7 @@ def __init__( if self.priority is None: self.priority = DEFAULT_PRIORITY - self.date_created = datetime.now() + self.date_created = datetime.now() # noqa: DTZ005 self._description = description if isinstance(identity_key, str): @@ -524,7 +524,7 @@ def perform(self): elif not self.max_retries: # infinite retries raise elif self.retry >= self.max_retries: - type_, value, traceback = sys.exc_info() + type_, value, _traceback = sys.exc_info() # change the exception type but keep the original # traceback and message: # http://blog.ianbicking.org/2007/09/12/re-raising-exceptions/ @@ -546,48 +546,45 @@ def in_temporary_env(self): finally: self._env = env - def _get_common_dependent_jobs_query(self): - return """ - UPDATE queue_job - SET state = %s - FROM ( - SELECT child.id, array_agg(parent.state) as parent_states - FROM queue_job job - JOIN LATERAL - json_array_elements_text( - job.dependencies::json->'reverse_depends_on' - ) child_deps ON true - JOIN queue_job child - ON child.graph_uuid = job.graph_uuid - AND child.uuid = child_deps - JOIN LATERAL - json_array_elements_text( - child.dependencies::json->'depends_on' - ) parent_deps ON true - JOIN queue_job parent - ON parent.graph_uuid = job.graph_uuid - AND parent.uuid = parent_deps - WHERE job.uuid = %s - GROUP BY child.id - ) jobs - WHERE - queue_job.id = jobs.id - AND %s = ALL(jobs.parent_states) - AND state = %s; - """ - def should_check_dependents(self): return any(self.__reverse_depends_on_uuids) def enqueue_waiting(self): - sql = self._get_common_dependent_jobs_query() - self.env.cr.execute(sql, (PENDING, self.uuid, DONE, WAIT_DEPENDENCIES)) - self.env["queue.job"].invalidate_model(["state"]) + dependents = ( + self.env["queue.job"] + .sudo() + .search( + [ + ("graph_uuid", "=", self.graph_uuid), + ("state", "=", WAIT_DEPENDENCIES), + ("dependency_ids", "any", [("uuid", "=", self.uuid)]), + ("dependency_ids", "not any", [("state", "!=", DONE)]), + ] + ) + ) + dependents.state = PENDING def cancel_dependent_jobs(self): - sql = self._get_common_dependent_jobs_query() - self.env.cr.execute(sql, (CANCELLED, self.uuid, CANCELLED, WAIT_DEPENDENCIES)) - self.env["queue.job"].invalidate_model(["state"]) + """Cancel the jobs waiting on this one, and their own dependents + + A dependent job can no longer run once one of its dependencies is + cancelled, whatever the state of the others. + """ + dependents = self.db_record().dependent_ids + while dependents: + dependents = dependents.filtered( + lambda dependent: dependent.state == WAIT_DEPENDENCIES + ) + dependents.write( + { + "state": CANCELLED, + "date_cancelled": self.date_cancelled, + "result": self.env._( + "Cancelled because a job it depends on was cancelled" + ), + } + ) + dependents = dependents.dependent_ids def store(self): """Store the Job""" @@ -745,9 +742,8 @@ def job_function_name(self): @property def identity_key(self): - if self._identity_key is None: - if self._identity_key_func: - self._identity_key = self._identity_key_func(self) + if self._identity_key is None and self._identity_key_func: + self._identity_key = self._identity_key_func(self) return self._identity_key @identity_key.setter @@ -808,9 +804,9 @@ def eta(self, value): if not value: self._eta = None elif isinstance(value, timedelta): - self._eta = datetime.now() + value + self._eta = datetime.now() + value # noqa: DTZ005 elif isinstance(value, int): - self._eta = datetime.now() + timedelta(seconds=value) + self._eta = datetime.now() + timedelta(seconds=value) # noqa: DTZ005 else: self._eta = value @@ -845,13 +841,13 @@ def set_pending(self, result=None, reset_retry=True): def set_enqueued(self): self.state = ENQUEUED - self.date_enqueued = datetime.now() + self.date_enqueued = datetime.now() # noqa: DTZ005 self.date_started = None self.worker_pid = None def set_started(self): self.state = STARTED - self.date_started = datetime.now() + self.date_started = datetime.now() # noqa: DTZ005 self.worker_pid = os.getpid() self.add_lock_record() @@ -859,13 +855,13 @@ def set_done(self, result=None): self.state = DONE self.exc_name = None self.exc_info = None - self.date_done = datetime.now() + self.date_done = datetime.now() # noqa: DTZ005 if result is not None: self.result = result def set_cancelled(self, result=None): self.state = CANCELLED - self.date_cancelled = datetime.now() + self.date_cancelled = datetime.now() # noqa: DTZ005 if result is not None: self.result = result @@ -925,7 +921,7 @@ def related_action(self): if not funcname: funcname = record._default_related_action if not isinstance(funcname, str): - raise ValueError( + raise ValueError( # noqa: TRY004 "related_action must be the name of the method on queue.job as string" ) action = getattr(record, funcname) diff --git a/queue_job/jobrunner/__init__.py b/queue_job/jobrunner/__init__.py index 50dd45e39d..c83c56b910 100644 --- a/queue_job/jobrunner/__init__.py +++ b/queue_job/jobrunner/__init__.py @@ -4,7 +4,6 @@ import logging from threading import Thread -import time from configparser import ConfigParser from odoo.service import server @@ -30,7 +29,7 @@ queue_job_config = dict(cp["queue_job"]) -from .runner import QueueJobRunner, _channels # noqa: E402 +from .runner import QueueJobRunner, _channels _logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ def __init__(self): def run(self): # sleep a bit to let the workers start at ease - time.sleep(START_DELAY) + self.runner.sleep(START_DELAY) self.runner.run() def stop(self): @@ -80,7 +79,7 @@ def process_work(self): self.runner = QueueJobRunner.from_environ_or_config() self._recover = False _logger.debug("WorkerJobRunner (%s) starting up", self.pid) - time.sleep(START_DELAY) + self.runner.sleep(START_DELAY) self.runner.run() def signal_time_expired_handler(self, n, stack): @@ -151,7 +150,8 @@ def prefork_worker_pop(server, pid): def threaded_start(server, *args, **kwargs): res = orig_threaded_start(server, *args, **kwargs) - _start_runner_thread("threaded server") + if not kwargs.get("stop"): + _start_runner_thread("threaded server") return res diff --git a/queue_job/jobrunner/channels.py b/queue_job/jobrunner/channels.py index bfc8984862..4f29e85acd 100644 --- a/queue_job/jobrunner/channels.py +++ b/queue_job/jobrunner/channels.py @@ -179,7 +179,7 @@ class ChannelJob: """ - __slots__ = ("db_name", "channel", "uuid", "_sorting_key", "__weakref__") + __slots__ = ("db_name", "channel", "uuid", "_sorting_key", "__weakref__") # noqa: RUF023 def __init__(self, db_name, channel, uuid, seq, date_created, priority, eta): self.db_name = db_name @@ -402,9 +402,14 @@ class Channel: with a capacity of 1. It is also possible to dedicate a channel with a limited capacity for application-autocreated subchannels without risking to overflow the system. + + A paused channel does not process any job until it is resumed. All subchannels + are blocked with their parent channel. """ - def __init__(self, name, parent, capacity=None, sequential=False, throttle=0): + def __init__( + self, name, parent, capacity=None, sequential=False, throttle=0, paused=False + ): self.name = name self.parent = parent if self.parent: @@ -417,6 +422,7 @@ def __init__(self, name, parent, capacity=None, sequential=False, throttle=0): self.capacity = capacity self.throttle = throttle # seconds self.sequential = sequential + self.paused = paused @property def sequential(self): @@ -434,11 +440,13 @@ def configure(self, config): * capacity * sequential * throttle + * paused """ assert self.fullname.endswith(config["name"]) self.capacity = config.get("capacity", None) self.sequential = bool(config.get("sequential", False)) self.throttle = int(config.get("throttle", 0)) + self.paused = bool(config.get("paused", False)) if self.sequential and self.capacity != 1: raise ValueError("A sequential channel must have a capacity of 1") @@ -455,9 +463,10 @@ def get_subchannel_by_name(self, subchannel_name): def __str__(self): capacity = "∞" if self.capacity is None else str(self.capacity) + paused = ",paused" if self.paused else "" return ( f"{self.fullname}(C:{capacity},Q:{len(self._queue)}," - f"R:{len(self._running)},F:{len(self._failed)})" + f"R:{len(self._running)},F:{len(self._failed)}{paused})" ) def remove(self, job): @@ -514,6 +523,8 @@ def set_failed(self, job): _logger.debug("job %s marked failed in channel %s", job.uuid, self) def has_capacity(self): + if self.paused: + return False if self.sequential and self._failed: # a sequential queue blocks on failed jobs return False @@ -796,6 +807,31 @@ class ChannelManager: >>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [] + + Test pausing a channel + + >>> cm = ChannelManager() + >>> cm.simple_configure('root:4,P:2:paused,P.sub:1') + >>> cm.notify(db, 'P', 'P1', 1, 0, 10, None, 'pending') + >>> cm.notify(db, 'P.sub', 'PS1', 2, 0, 10, None, 'pending') + + Paused channel yields no job + + >>> pp(list(cm.get_jobs_to_run(now=100))) + [] + + Resuming the channel yields the pending jobs + + >>> cm.simple_configure('root:4,P:2') + >>> pp(list(cm.get_jobs_to_run(now=100))) + [, ] + + Pausing the root channel blocks everything + + >>> cm.simple_configure('root:4:paused') + >>> cm.notify(db, 'P', 'P3', 4, 0, 10, None, 'pending') + >>> pp(list(cm.get_jobs_to_run(now=106))) + [] """ def __init__(self): diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index 6ff2e2f260..79ccf8a787 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -31,6 +31,7 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT import odoo +from odoo.modules.db import list_dbs from odoo.tools import config from . import queue_job_config @@ -72,7 +73,7 @@ def _odoo_now(): def _connection_info_for(db_name): - db_or_uri, connection_info = odoo.sql_db.connection_info_for(db_name) + _db_or_uri, connection_info = odoo.sql_db.connection_info_for(db_name) for p in ("host", "port", "user", "password"): cfg = os.environ.get( @@ -139,7 +140,7 @@ def close(self): # del try: self.conn.close() - except Exception: + except Exception: # noqa: BLE001, S110 pass self.conn = None @@ -335,6 +336,7 @@ def __init__( self.channel_manager.simple_configure(channel_config_string) self.db_by_name = {} self._stop = False + self._stop_event = threading.Event() self._stop_pipe = os.pipe() def __del__(self): @@ -382,7 +384,7 @@ def get_db_names(self): db_names = config["db_name"] if db_names: return db_names - return odoo.service.db.list_dbs(True) + return list_dbs(force=True) def close_databases(self, remove_jobs=True): for db_name, db in self.db_by_name.items(): @@ -474,7 +476,7 @@ def wait_notification(self): # if timeout remains a large negative number, it is most # probably a bug _logger.debug("select() timeout: %.2f sec", timeout) - if timeout > 0: + if timeout > 0: # noqa: SIM102 if conns and not self._stop: with select() as sel: for conn in conns: @@ -486,9 +488,18 @@ def wait_notification(self): continue key.fileobj.poll() + def sleep(self, seconds): + """Wait for ``seconds``, or less if :meth:`stop` is called meanwhile. + + A plain ``time.sleep`` would hold the thread, and so the server + shutdown that joins it, for up to the whole delay. + """ + self._stop_event.wait(seconds) + def stop(self): _logger.info("graceful stop requested") self._stop = True + self._stop_event.set() # wakeup the select() in wait_notification os.write(self._stop_pipe[1], b".") @@ -520,12 +531,12 @@ def run(self): ERROR_RECOVERY_DELAY, ) self.close_databases() - time.sleep(ERROR_RECOVERY_DELAY) + self.sleep(ERROR_RECOVERY_DELAY) except Exception: _logger.exception( "exception: sleeping %ds and retrying", ERROR_RECOVERY_DELAY ) self.close_databases() - time.sleep(ERROR_RECOVERY_DELAY) + self.sleep(ERROR_RECOVERY_DELAY) self.close_databases(remove_jobs=False) _logger.info("stopped") diff --git a/queue_job/migrations/19.0.2.2.0/pre-migration.py b/queue_job/migrations/19.0.2.2.0/pre-migration.py new file mode 100644 index 0000000000..a9150c45d7 --- /dev/null +++ b/queue_job/migrations/19.0.2.2.0/pre-migration.py @@ -0,0 +1,11 @@ +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) +from openupgradelib import openupgrade + + +@openupgrade.migrate() +def migrate(env, version): + # Old jobs are now removed by the base autovacuum cron + openupgrade.delete_records_safely_by_xml_id( + env, + ["queue_job.ir_cron_autovacuum_queue_jobs"], + ) diff --git a/queue_job/models/queue_job.py b/queue_job/models/queue_job.py index 23cf5de370..cc5fbf9ebd 100644 --- a/queue_job/models/queue_job.py +++ b/queue_job/models/queue_job.py @@ -4,10 +4,10 @@ import logging import random import time -from datetime import datetime, timedelta from odoo import api, exceptions, fields, models -from odoo.tools import config, html_escape +from odoo.fields import Domain +from odoo.tools import SQL, html_escape from odoo.tools.sql import create_index from odoo.addons.base_sparse_field.models.fields import Serialized @@ -35,7 +35,7 @@ class QueueJob(models.Model): _name = "queue.job" _description = "Queue Job" - _inherit = ["mail.thread", "mail.activity.mixin"] + _inherit = ["mail.thread", "mail.activity.mixin"] # noqa: RUF012 _log_access = False _order = "date_created DESC, date_done DESC" @@ -70,9 +70,7 @@ class QueueJob(models.Model): help="Single shared identifier of a Graph. Empty for a single job.", ) user_id = fields.Many2one(comodel_name="res.users", string="User ID") - company_id = fields.Many2one( - comodel_name="res.company", string="Company", index=True - ) + company_id = fields.Many2one(comodel_name="res.company", index=True) name = fields.Char(string="Description", readonly=True) model_name = fields.Char(string="Model", readonly=True) @@ -83,6 +81,20 @@ class QueueJob(models.Model): base_type=models.BaseModel, ) dependencies = Serialized(readonly=True) + dependency_ids = fields.Many2many( + comodel_name="queue.job", + compute="_compute_dependency_ids", + search="_search_dependency_ids", + string="Dependency Jobs", + help="Jobs this job depends on.", + ) + dependent_ids = fields.Many2many( + comodel_name="queue.job", + compute="_compute_dependency_ids", + search="_search_dependent_ids", + string="Dependent Jobs", + help="Jobs depending on this job.", + ) # dependency graph as expected by the field widget dependency_graph = Serialized(compute="_compute_dependency_graph") graph_jobs_count = fields.Integer(compute="_compute_graph_jobs_count") @@ -121,7 +133,6 @@ class QueueJob(models.Model): channel_method_name = fields.Char(string="Complete Method Name", readonly=True) job_function_id = fields.Many2one( comodel_name="queue.job.function", - string="Job Function", readonly=True, ) @@ -144,7 +155,7 @@ def init(self): ), comment=("Queue Job: partial index for identity_key on active states"), ) - # Used by .autovacuum + # Used by ._gc_old_jobs create_index( cr, "queue_job_channel_date_done_date_created_index", @@ -153,6 +164,55 @@ def init(self): comment="Queue Job: index to accelerate autovacuum", ) + @api.depends("dependencies") + def _compute_dependency_ids(self): + uuids = set() + for record in self: + dependencies = record.dependencies or {} + uuids.update(dependencies.get("depends_on", [])) + uuids.update(dependencies.get("reverse_depends_on", [])) + jobs = self.search([("uuid", "in", list(uuids))]) + jobs_by_uuid = jobs.grouped("uuid") + for record in self: + dependencies = record.dependencies or {} + record.dependency_ids = self.browse().concat( + jobs_by_uuid.get(uuid, self.browse()) + for uuid in dependencies.get("depends_on", []) + ) + record.dependent_ids = self.browse().concat( + jobs_by_uuid.get(uuid, self.browse()) + for uuid in dependencies.get("reverse_depends_on", []) + ) + + def _search_dependency_ids(self, operator, value): + return self._search_dependencies("depends_on", operator, value) + + def _search_dependent_ids(self, operator, value): + return self._search_dependencies("reverse_depends_on", operator, value) + + def _search_dependencies(self, key, operator, value): + """Search the jobs having one of the given jobs in ``dependencies[key]``""" + if operator == "in": + operator, value = "any", Domain("id", "in", value) + if operator != "any": + return NotImplemented + jobs = self._search(value) + return Domain.custom( + to_sql=lambda table: SQL( + """ + EXISTS ( + SELECT 1 + FROM json_array_elements_text(%s::json -> %s) AS dep(uuid) + JOIN queue_job AS dep_job ON dep_job.uuid = dep.uuid + WHERE dep_job.id IN %s + ) + """, + table.dependencies, + key, + jobs.subselect(), + ) + ) + @api.depends("dependencies") def _compute_dependency_graph(self): graph_uuids = [uuid for uuid in self.mapped("graph_uuid") if uuid] @@ -342,8 +402,11 @@ def button_cancelled(self): # If job was set to DONE do not cancel it states_from = (WAIT_DEPENDENCIES, PENDING, ENQUEUED, FAILED) result = self.env._("Cancelled by %s", self.env.user.name) - records = self.filtered(lambda job_: job_.state in states_from) - records._change_job_state(CANCELLED, result=result) + for record in self: + # Cancelling a job also cancels its dependents, some of them may + # be in self already + if record.state in states_from: + record._change_job_state(CANCELLED, result=result) return True def requeue(self): @@ -374,7 +437,7 @@ def _subscribe_users_domain(self): if not group: return None companies = self.mapped("company_id") - domain = [("group_ids", "=", group.id)] + domain = [("all_group_ids", "in", group.ids)] if companies: domain.append(("company_id", "in", companies.ids)) return domain @@ -408,33 +471,41 @@ def _needaction_domain_get(self): """ return [("state", "=", "failed")] - def autovacuum(self): - """Delete all jobs done based on the removal interval defined on the - channel + @api.autovacuum + def _gc_old_jobs(self): + """Delete the jobs done or cancelled before the removal interval of + their channel, in batches of 1000. + + Jobs of an unknown channel (renamed, deleted) use the removal interval + of the root channel. - Called from a cron. + Called from the base autovacuum cron, which commits after each batch + and calls it again while it returns remaining jobs. """ - for channel in self.env["queue.job.channel"].search([]): # pylint: disable=no-search-all - deadline = datetime.now() - timedelta(days=int(channel.removal_interval)) - # Delete in chunks using a stable order (matches composite index) - while True: - jobs = self.search( - [ - "|", - ("date_done", "<=", deadline), - ("date_cancelled", "<=", deadline), - ("channel", "=", channel.complete_name), - ], - order="date_done, date_created", - limit=1000, - ) - if jobs: - jobs.sudo().unlink() - if not config["test_enable"]: - self.env.cr.commit() # pylint: disable=E8102 - else: - break - return True + channels = self.env["queue.job.channel"].search([]) # pylint: disable=no-search-all + root = self.env.ref("queue_job.channel_root") + intervals = [ + (Domain("channel", "=", channel.complete_name), channel.removal_interval) + for channel in channels + ] + intervals.append( + ( + Domain("channel", "not in", channels.mapped("complete_name")), + root.removal_interval, + ) + ) + domain = Domain.OR( + channel_domain + & ( + Domain("date_done", "<=", f"-{days}d") + | Domain("date_cancelled", "<=", f"-{days}d") + ) + for channel_domain, days in intervals + ) + limit = 1000 + jobs = self.search(domain, order="date_done, date_created", limit=limit + 1) + jobs[:limit].sudo().unlink() + return len(jobs[:limit]), len(jobs) > limit def related_action_open_record(self): """Open a form view with the record(s) of the job. diff --git a/queue_job/models/queue_job_channel.py b/queue_job/models/queue_job_channel.py index 0a5c1c917b..ee5f7588fe 100644 --- a/queue_job/models/queue_job_channel.py +++ b/queue_job/models/queue_job_channel.py @@ -37,7 +37,7 @@ def _compute_complete_name(self): if not record.name: complete_name = "" # new record elif record.parent_id: - complete_name = ".".join([record.parent_id.complete_name, record.name]) + complete_name = f"{record.parent_id.complete_name}.{record.name}" else: complete_name = record.name record.complete_name = complete_name diff --git a/queue_job/models/queue_job_function.py b/queue_job/models/queue_job_function.py index 872f5831f0..d574753593 100644 --- a/queue_job/models/queue_job_function.py +++ b/queue_job/models/queue_job_function.py @@ -6,7 +6,7 @@ import re from collections import namedtuple -from odoo import api, exceptions, fields, models, tools +from odoo import api, exceptions, fields, models from ..fields import JobSerialized @@ -45,9 +45,7 @@ def _default_channel(self): # model and method should be required, but the required flag doesn't # let a chance to _inverse_name to be executed - model_id = fields.Many2one( - comodel_name="ir.model", string="Model", ondelete="cascade" - ) + model_id = fields.Many2one(comodel_name="ir.model", ondelete="cascade") method = fields.Char() on_fail_method = fields.Char( help="Model function to be called if the job is failed and will not be " @@ -56,7 +54,6 @@ def _default_channel(self): channel_id = fields.Many2one( comodel_name="queue.job.channel", - string="Channel", required=True, default=lambda r: r._default_channel(), ) @@ -187,7 +184,7 @@ def _parse_retry_pattern(self): retry_pattern = {} return retry_pattern - @tools.ormcache("name") + @api.ormcache("name") def job_config(self, name): config = self.search([("name", "=", name)], limit=1) if not config: @@ -279,15 +276,15 @@ def create(self, vals_list): new_vals_list.append(vals) vals_list = new_vals_list records |= super().create(vals_list) - self.env.registry.clear_cache() + self.env.transaction.invalidate_ormcache() return records def write(self, values): res = super().write(values) - self.env.registry.clear_cache() + self.env.transaction.invalidate_ormcache() return res def unlink(self): res = super().unlink() - self.env.registry.clear_cache() + self.env.transaction.invalidate_ormcache() return res diff --git a/queue_job/security/ir.access.csv b/queue_job/security/ir.access.csv new file mode 100644 index 0000000000..ae56a64612 --- /dev/null +++ b/queue_job/security/ir.access.csv @@ -0,0 +1,9 @@ +id,name,model_id,group_id/id,operation,domain +access_queue_job_manager,queue job manager,queue.job,queue_job.group_queue_job_manager,ru, +queue_job_comp_rule,Job Queue multi-company,queue.job,,crud,"['|', ('company_id', '=', False), ('company_id', 'in', company_ids)]" +access_queue_job_lock_manager,queue job lock manager,queue.job.lock,queue_job.group_queue_job_manager,r, +access_queue_job_function_manager,queue job functions manager,queue.job.function,queue_job.group_queue_job_manager,crud, +access_queue_job_channel_manager,queue job channel manager,queue.job.channel,queue_job.group_queue_job_manager,crud, +access_queue_requeue_job,queue requeue job manager,queue.requeue.job,queue_job.group_queue_job_manager,crud, +access_queue_jobs_to_done,queue jobs to done manager,queue.jobs.to.done,queue_job.group_queue_job_manager,crud, +access_queue_jobs_to_cancelled,queue jobs to cancelled manager,queue.jobs.to.cancelled,queue_job.group_queue_job_manager,crud, diff --git a/queue_job/security/ir.model.access.csv b/queue_job/security/ir.model.access.csv deleted file mode 100644 index 9a451d6a78..0000000000 --- a/queue_job/security/ir.model.access.csv +++ /dev/null @@ -1,8 +0,0 @@ -id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink -access_queue_job_manager,queue job manager,queue_job.model_queue_job,queue_job.group_queue_job_manager,1,1,0,0 -access_queue_job_lock_manager,queue job lock manager,queue_job.model_queue_job_lock,queue_job.group_queue_job_manager,1,0,0,0 -access_queue_job_function_manager,queue job functions manager,queue_job.model_queue_job_function,queue_job.group_queue_job_manager,1,1,1,1 -access_queue_job_channel_manager,queue job channel manager,queue_job.model_queue_job_channel,queue_job.group_queue_job_manager,1,1,1,1 -access_queue_requeue_job,queue requeue job manager,queue_job.model_queue_requeue_job,queue_job.group_queue_job_manager,1,1,1,1 -access_queue_jobs_to_done,queue jobs to done manager,queue_job.model_queue_jobs_to_done,queue_job.group_queue_job_manager,1,1,1,1 -access_queue_jobs_to_cancelled,queue jobs to cancelled manager,queue_job.model_queue_jobs_to_cancelled,queue_job.group_queue_job_manager,1,1,1,1 diff --git a/queue_job/security/security.xml b/queue_job/security/security.xml index 740226988e..b4bf2549e9 100644 --- a/queue_job/security/security.xml +++ b/queue_job/security/security.xml @@ -1,31 +1,19 @@ - + - - - Job Queue - 50 - - - Job Queue - 20 - - - Job Queue Manager - - - - - - - Job Queue multi-company - - - ['|', ('company_id', '=', False), ('company_id', 'in', company_ids)] - - + + Job Queue + + + + Job Queue + + + + Job Queue Manager + + + diff --git a/queue_job/static/description/index.html b/queue_job/static/description/index.html index 58b23819d8..c6f79d66ea 100644 --- a/queue_job/static/description/index.html +++ b/queue_job/static/description/index.html @@ -3,7 +3,7 @@ -README.rst +Job Queue