From a8287aeb4becacca775f29acf269111cff9e361b Mon Sep 17 00:00:00 2001 From: sanjanch Date: Tue, 23 Dec 2025 13:14:00 +0000 Subject: [PATCH 1/2] Added script Pending Contract Check and its pytest files --- aci-preupgrade-validation-script.py | 30 ++++++++++ docs/docs/validations.md | 15 ++++- .../fvPndgCtrct_count_not_zero.json | 12 ++++ .../fvPndgCtrct_count_zero.json | 12 ++++ .../test_pending_contract_check.py | 57 +++++++++++++++++++ 5 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 tests/checks/pending_contract_check/fvPndgCtrct_count_not_zero.json create mode 100644 tests/checks/pending_contract_check/fvPndgCtrct_count_zero.json create mode 100644 tests/checks/pending_contract_check/test_pending_contract_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ebe0477..11968a1 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6007,6 +6007,35 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title='Pending Contract Check') +def pending_contract_check(tversion, **kwargs): + result = PASS + headers = ["Pending Contract Count"] + data = [] + recommended_action = 'Pending contracts detected. Contact TAC to resolve stuck contract deployments before upgrade to prevent outage.' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#pending-contract-check' + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + if tversion.older_than('6.1(4h)'): + # Query fvPndgCtrct with count + fvPndgCtrct_count = icurl('class', 'fvPndgCtrct.json?rsp-subtree-include=count') + + if fvPndgCtrct_count: + count = int(fvPndgCtrct_count[0]['moCount']['attributes']['count']) + + if count > 0: + result = FAIL_O + data.append([str(count)]) + + else: + return Result(result=NA, msg=VER_NOT_AFFECTED) + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6168,6 +6197,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + pending_contract_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index fa1fc0e..043a74a 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -191,6 +191,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[Pending Contract Check][d29] | CSCwp99433 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -220,6 +221,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash +[d29]: #pending-contract-check ## General Check Details @@ -2227,9 +2229,9 @@ A shared service (VRF Route Leaking) contract with vzAny as a consumer may use m When Rule Expansion takes place after an upgrade, the increase in the TCAM space may result in TCAM overflow which can lead to traffic disruption because contracts that used to work may stop working. -See [Inter-VRF contract with vzAny as the consumer][60] in Cisco ACI Contract Guide for details about Rule Expansion and calculate the potential TCAM space usage when the Rule Expansion takes place. If there is a risk of TCAM overflow, consider enabling the policy compression directive on contract filters to mitigate the increase of TCAM usage. However, note that enabling the policy compression directive will result in loss of the statistics capability for those rules. Also, note that policy compression for contracts with PBR is supported only from 6.1(4). +See [Inter-VRF contract with vzAny as the consumer][61] in Cisco ACI Contract Guide for details about Rule Expansion and calculate the potential TCAM space usage when the Rule Expansion takes place. If there is a risk of TCAM overflow, consider enabling the policy compression directive on contract filters to mitigate the increase of TCAM usage. However, note that enabling the policy compression directive will result in loss of the statistics capability for those rules. Also, note that policy compression for contracts with PBR is supported only from 6.1(4). -See [Enable Policy Compression in Cisco ACI Contract Guide][61] for details about Policy Compression. +See [Enable Policy Compression in Cisco ACI Contract Guide][62] for details about Policy Compression. ## Defect Check Details @@ -2613,6 +2615,14 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### Pending Contract Check + +Due to [CSCwp99433][62], failed transactions during high-scale contract processing can cause pending contracts (`fvPndgCtrct`) to get permanently stuck in memory, preventing contract policies from being deployed to leaf switches. The issue occurs when performing operations that require significant contract reprocessing, such as changing VRF enforcement mode (enforced ↔ unenforced) for VRFs containing hundreds or thousands of contracts, bulk contract configuration changes, or large-scale tenant migrations. When triggered, the `policyelement` service fails multiple transactions but continues processing, leaving `fvPndgCtrct` objects stuck in the system. + +This script checks for the presence of pending contracts by querying `fvPndgCtrct` objects with a count in the affected version. + +If any instances of `fvPndgCtrct` are flagged by this script, Cisco TAC must be contacted to identify and resolve the stuck contracts. + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html @@ -2676,3 +2686,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression +[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp99433 diff --git a/tests/checks/pending_contract_check/fvPndgCtrct_count_not_zero.json b/tests/checks/pending_contract_check/fvPndgCtrct_count_not_zero.json new file mode 100644 index 0000000..ca43089 --- /dev/null +++ b/tests/checks/pending_contract_check/fvPndgCtrct_count_not_zero.json @@ -0,0 +1,12 @@ +[ + { + "moCount": { + "attributes": { + "childAction": "", + "count": "10", + "dn": "", + "status": "" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/pending_contract_check/fvPndgCtrct_count_zero.json b/tests/checks/pending_contract_check/fvPndgCtrct_count_zero.json new file mode 100644 index 0000000..ce55fe6 --- /dev/null +++ b/tests/checks/pending_contract_check/fvPndgCtrct_count_zero.json @@ -0,0 +1,12 @@ +[ + { + "moCount": { + "attributes": { + "childAction": "", + "count": "0", + "dn": "", + "status": "" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/pending_contract_check/test_pending_contract_check.py b/tests/checks/pending_contract_check/test_pending_contract_check.py new file mode 100644 index 0000000..094b9e5 --- /dev/null +++ b/tests/checks/pending_contract_check/test_pending_contract_check.py @@ -0,0 +1,57 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "pending_contract_check" + +# icurl queries +pending_contract_api = 'fvPndgCtrct.json' +pending_contract_api += '?rsp-subtree-include=count' + + +@pytest.mark.parametrize( + "icurl_outputs, tversion, expected_result", + [ + # NO TVERSION PROVIDED - MANUAL CHECK + ( + {pending_contract_api: read_data(dir, "fvPndgCtrct_count_zero.json")}, + None, + script.MANUAL, + ), + # TVERSION > 6.1(3g) and no pending contracts - NA + ( + {pending_contract_api: read_data(dir, "fvPndgCtrct_count_zero.json")}, + "6.1(5a)", + script.NA, + ), + # TVERSION > 6.1(3g) and pending contracts exist - NA + ( + {pending_contract_api: read_data(dir, "fvPndgCtrct_count_not_zero.json")}, + "6.1(5a)", + script.NA, + ), + # TVERSION < 6.1(3g) and no pending contracts - PASS + ( + {pending_contract_api: read_data(dir, "fvPndgCtrct_count_zero.json")}, + "5.2(1a)", + script.PASS, + ), + # TVERSION < 6.1(3g) and pending contracts exist - FAIL_O + ( + {pending_contract_api: read_data(dir, "fvPndgCtrct_count_not_zero.json")}, + "5.2(1a)", + script.FAIL_O, + ), + + ], +) +def test_logic(run_check, mock_icurl, icurl_outputs, tversion, expected_result): + result = run_check(tversion=script.AciVersion(tversion) if tversion else None) + assert result.result == expected_result \ No newline at end of file From 719f9079b7f78eb96227641c5826e7fccff8c9da Mon Sep 17 00:00:00 2001 From: sanjanch Date: Tue, 23 Dec 2025 13:35:06 +0000 Subject: [PATCH 2/2] Modified the validation.md file --- docs/docs/validations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 043a74a..f28cbbd 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2229,9 +2229,9 @@ A shared service (VRF Route Leaking) contract with vzAny as a consumer may use m When Rule Expansion takes place after an upgrade, the increase in the TCAM space may result in TCAM overflow which can lead to traffic disruption because contracts that used to work may stop working. -See [Inter-VRF contract with vzAny as the consumer][61] in Cisco ACI Contract Guide for details about Rule Expansion and calculate the potential TCAM space usage when the Rule Expansion takes place. If there is a risk of TCAM overflow, consider enabling the policy compression directive on contract filters to mitigate the increase of TCAM usage. However, note that enabling the policy compression directive will result in loss of the statistics capability for those rules. Also, note that policy compression for contracts with PBR is supported only from 6.1(4). +See [Inter-VRF contract with vzAny as the consumer][60] in Cisco ACI Contract Guide for details about Rule Expansion and calculate the potential TCAM space usage when the Rule Expansion takes place. If there is a risk of TCAM overflow, consider enabling the policy compression directive on contract filters to mitigate the increase of TCAM usage. However, note that enabling the policy compression directive will result in loss of the statistics capability for those rules. Also, note that policy compression for contracts with PBR is supported only from 6.1(4). -See [Enable Policy Compression in Cisco ACI Contract Guide][62] for details about Policy Compression. +See [Enable Policy Compression in Cisco ACI Contract Guide][61] for details about Policy Compression. ## Defect Check Details