Skip to content

gpcheckcat: fix the defects the Behave suite exposes #2182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MisterRaindrop wants to merge 1 commit into
base: main
Choose a base branch
from
+454 −1
Open

gpcheckcat: fix the defects the Behave suite exposes #2182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 215 additions & 1 deletion gpMgmt/bin/gpcheckcat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1075,9 +1075,12 @@ def checkOwners():
a.rolname, m.rolname as coordinator_rolname
from gp_dist_random('pg_class') r
join pg_class c on (c.oid = r.oid)
left join pg_index i on (c.oid = i.indexrelid)
left join pg_appendonly ao on (c.oid = ao.segrelid or
c.oid = ao.blkdirrelid or
c.oid = ao.blkdiridxid)
c.oid = ao.visimaprelid or
i.indrelid = ao.blkdirrelid or
i.indrelid = ao.visimaprelid)
left join pg_class o on (o.oid = ao.relid or
o.reltoastrelid = c.oid)
join pg_authid a on (a.oid = r.relowner)
Expand Down Expand Up @@ -1948,6 +1951,207 @@ def checkOrphanedToastTables():
issue_type="orphaned_toast_tables",
description='Repairing orphaned TOAST tables')

# The mix_distribution_policy check below -- fetch_guc_value(),
# generateDistPolicyQueryFile(), checkMixDistPolicy() and their helpers --
# is cherry-picked from Greenplum Database (greenplum-db/gpdb-archive @ 482967c1b4),
# gpMgmt/bin/gpcheckcat.
def fetch_guc_value(guc):
qry = '''
show {}
'''.format(guc)
try:
conn = connect2(GV.cfg[GV.coordinator_dbid])
curs = conn.query(qry)
rows = curs.getresult()
guc_value = rows[0][0]
return guc_value

except Exception as e:
setError(ERROR_NOREPAIR)
GV.checkStatus = False
myprint('[ERROR] executing test: mix_distribution_policy')
myprint(' Execution error: ' + str(e))

def generateDistPolicyQueryFile():

query_sql = '''
-- all tables that use legacy policy:
with legacy_opclass_oids(oid_array) as (
select
array_agg(oid)
from
pg_opclass
where
opcfamily in (
select
amprocfamily
from
pg_amproc
where
amproc :: oid in (
6140, 6141, 6142, 6143, 6144, 6145, 6146,
6147, 6148, 6149, 6150, 6151, 6152,
6153, 6154, 6155, 6156, 6157, 6158,
6159, 6160, 6161, 6162, 6163, 6164,
6165, 6166, 6167, 6168, 6170, 6169,
6171
)
)
)
select
localoid :: regclass :: text as "Legacy Policy"
from
gp_distribution_policy,
legacy_opclass_oids
where
policytype = 'p'
and distclass :: oid[] && oid_array;

-- all tables that don't use any legacy policy:
with legacy_opclass_oids(oid_array) as (
select
array_agg(oid)
from
pg_opclass
where
opcfamily in (
select
amprocfamily
from
pg_amproc
where
amproc :: oid in (
6140, 6141, 6142, 6143, 6144, 6145, 6146,
6147, 6148, 6149, 6150, 6151, 6152,
6153, 6154, 6155, 6156, 6157, 6158,
6159, 6160, 6161, 6162, 6163, 6164,
6165, 6166, 6167, 6168, 6170, 6169,
6171
)
)
)
select
localoid :: regclass :: text as "Non Legacy Policy"
from
gp_distribution_policy,
legacy_opclass_oids
where
policytype = 'p'
and not (distclass :: oid[] && oid_array);
'''
filename = 'gpcheckcat.distpolicy.sql'

if not os.path.exists(filename) :
try:
with open(filename, 'w') as fp:
fp.write(query_sql + "\n")
except Exception as e:
logger.warning('Unable to generate verify file for {}'.format(filename))


# Test to check if there are tables that use both legacy opclass/non legacy opclass
# in distribution policy
def checkMixDistPolicy() :

qry = '''
with legacy_opclass_oids(oid_array) as (
select
array_agg(oid)
from
pg_opclass
where
opcfamily in (
select
amprocfamily
from
pg_amproc
where
amproc :: oid in (
6140, 6141, 6142, 6143, 6144, 6145, 6146,
6147, 6148, 6149, 6150, 6151, 6152,
6153, 6154, 6155, 6156, 6157, 6158,
6159, 6160, 6161, 6162, 6163, 6164,
6165, 6166, 6167, 6168, 6170, 6169,
6171
)
)
),
all_hash_ops(dc) as (
select
distinct unnest(distclass :: oid[])
from
gp_distribution_policy
)
select
count(1) filter(
where
array[x.dc] && oid_array
) as n_legacy_dist_class,
count(1) as n_total_dist_class
from
all_hash_ops x,
legacy_opclass_oids y;
'''

try:
conn = connect2(GV.cfg[GV.coordinator_dbid])
curs = conn.query(qry)
rows = curs.getresult()

if rows:
row = rows[0]
n_legacy_dist_class = row[0]
n_total_dist_class = row[1]
GV.checkStatus = False

if n_legacy_dist_class > 0 and n_total_dist_class > n_legacy_dist_class :
generateDistPolicyQueryFile()
#if this condition is true then we have mix distribution Policy
myprint(
'[ERROR]: Found tables created using both legacy and non legacy hashops'
' in distribution policy.'
'Please run the gpcheckcat.distpolicy.sql file to list the tables.'
)
else:
if (n_legacy_dist_class == 0 or n_legacy_dist_class == n_total_dist_class):
#if this condition is true then we dont have mix distribution policy
gp_use_legacy_hashops = fetch_guc_value("gp_use_legacy_hashops")
printDistPolicyMsg(gp_use_legacy_hashops,
n_legacy_dist_class,
n_total_dist_class
)

except Exception as e:
setError(ERROR_NOREPAIR)
GV.checkStatus = False
myprint('[ERROR] executing test: mix_distribution_policy')
myprint(' Execution error: ' + str(e))

def printDistPolicyMsg(gp_use_legacy_hashops,n_legacy_dist_class, n_total_dist_class):

GV.checkStatus = True

if n_total_dist_class - n_legacy_dist_class > 0 and gp_use_legacy_hashops == "on":
myprint(
'[ERROR]: GUC gp_use_legacy_hashops is on.'
' all newly created tables will use legacy hash ops by default for hash distributed table, '
'but there are tables using non-legacy hash ops in the cluster. '
'Please run the gpcheckcat.distpolicy.sql file to list the tables.'
)
GV.checkStatus = False

elif n_legacy_dist_class == 0 and gp_use_legacy_hashops == "off":
GV.checkStatus = True

elif n_legacy_dist_class > 0 and gp_use_legacy_hashops == "off" :
myprint(
'[ERROR]: GUC gp_use_legacy_hashops is off.'
' all newly created tables will use non legacy hash ops by default for hash distributed table, '
'but there are tables using legacy hash ops in the cluster. '
'Please run the gpcheckcat.distpolicy.sql file to list the tables.'
)
GV.checkStatus = False


############################################################################
# Help populating repair part for all checked types
Expand Down Expand Up @@ -2082,7 +2286,16 @@ all_checks = {
"version": 'main',
"order": 15,
"online": False
},
"mix_distribution_policy":
{
"description": "Check for tables that use legacy opclass in distribution policy",
"fn": lambda: checkMixDistPolicy(),
"version": 'main',
"order": 17,
"online": True
}

}


Expand Down Expand Up @@ -2366,6 +2579,7 @@ TableMainColumn['pg_type_encoding'] = ['typid', 'pg_type']
TableMainColumn['pg_window'] = ['winfnoid', 'pg_proc']
TableMainColumn['pg_password_history'] = ['passhistroleid', 'pg_authid']
TableMainColumn['pg_description'] = ['objoid', 'pg_description']
TableMainColumn['pg_shdescription'] = ['objoid', 'pg_shdescription']

# Table with OID (special case), these OIDs are known to be inconsistent
TableMainColumn['pg_attrdef'] = ['adrelid', 'pg_class']
Expand Down
Loading
Loading