Use google groups for CCs in OSS-Fuzz issue tracker

src/clusterfuzz/_internal/base/external_users.py

@@ -15,13 +15,17 @@
 
 from clusterfuzz._internal.base import memoize
 from clusterfuzz._internal.base import utils
+from clusterfuzz._internal.config import local_config
 from clusterfuzz._internal.datastore import data_handler
 from clusterfuzz._internal.datastore import data_types
 from clusterfuzz._internal.datastore import fuzz_target_utils
 from clusterfuzz._internal.datastore import ndb_utils
 
 MEMCACHE_TTL_IN_SECONDS = 15 * 60
 
+# Issue tracker CC group suffix
+CC_GROUP_SUFFIX = '-ccs'
+
 
 def _fuzzers_for_job(job_type, include_parents):
   """Return all fuzzers that have the job associated.
@@ -198,7 +202,10 @@ def _allowed_users_for_entity(name, entity_kind, auto_cc=None):
   return sorted(allowed_users)
 
 
-def _cc_users_for_entity(name, entity_type, security_flag):
+def _cc_users_for_entity(name,
+                         entity_type,
+                         security_flag,
+                         allow_cc_group_for_job=True):
   """Return CC users for entity."""
   users = _allowed_users_for_entity(name, entity_type,
                                     data_types.AutoCCType.ALL)
@@ -208,6 +215,20 @@ def _cc_users_for_entity(name, entity_type, security_flag):
         _allowed_users_for_entity(name, entity_type,
                                   data_types.AutoCCType.SECURITY))
 
+  if (entity_type != data_types.PermissionEntityKind.JOB or
+      not allow_cc_group_for_job):
+    return sorted(users)
+
+  # CC group is only available for jobs, as it is not possible to infer the
+  # project from the other permission entity kinds alone.
+  users_in_cc_group = _allowed_users_for_entity(
+      name, entity_type, data_types.AutoCCType.USE_CC_GROUP)
+  if users_in_cc_group:
+    # Assume users are synced with the project group.
+    group_name = get_cc_group_from_job(name)
+    if group_name:
+      users.append(group_name)
+
   return sorted(users)
 
 
@@ -336,15 +357,33 @@ def is_upload_allowed_for_user(user_email):
   return bool(permissions.get())
 
 
-def cc_users_for_job(job_type, security_flag):
+def cc_users_for_job(job_type, security_flag, allow_cc_group=True):
   """Return external users that should be CC'ed according to the given rule.
 
   Args:
     job_type: The name of the job
     security_flag: Whether or not the CC is for a security issue.
+    allow_cc_group: Whether to allow including the project cc group from the
+      job, if exists any user with the use cc group auto_cc type.
 
   Returns:
     A list of user emails that should be CC'ed.
   """
   return _cc_users_for_entity(job_type, data_types.PermissionEntityKind.JOB,
-                              security_flag)
+                              security_flag, allow_cc_group)
+
+
+def get_cc_group_from_job(job_type: str) -> str | None:
+  """Retrieve the job's project issue cc google group."""
+  project_name = data_handler.get_project_name(job_type)
+  if not project_name:
+    return None
+  return get_project_cc_group(project_name)
+
+
+def get_project_cc_group(project_name: str) -> str:
+  """Return issue tracker CC group email for a project."""
+  group_domain = local_config.ProjectConfig().get('issue_cc_groups.domain')
+  if not group_domain:
+    group_domain = 'google.com'
+  return f'{project_name}{CC_GROUP_SUFFIX}@{group_domain}'

src/clusterfuzz/_internal/base/feature_flags.py

@@ -35,6 +35,8 @@ class FeatureFlags(Enum):
 
   PREPROCESS_QUEUE_SIZE_LIMIT = 'preprocess_queue_size_limit'
 
+  UPDATE_OSS_FUZZ_USERS_AUTO_CC = 'update_oss_fuzz_users_auto_cc'
+
   @property
   def flag(self):
     """Get the feature flag."""

src/clusterfuzz/_internal/cron/oss_fuzz_build_status.py

@@ -20,6 +20,7 @@
 from google.cloud import ndb
 import requests
 
+from clusterfuzz._internal.base import external_users
 from clusterfuzz._internal.base import utils
 from clusterfuzz._internal.datastore import data_types
 from clusterfuzz._internal.issue_management import issue_tracker_policy
@@ -136,7 +137,8 @@ def get_build_time(build):
       stripped_timestamp.group(0), TIMESTAMP_FORMAT)
 
 
-def file_bug(issue_tracker, policy, project_name, build_id, ccs, build_type):
+def file_bug(issue_tracker, policy, project_name, build_id, project_cc_group,
+             build_type):
   """File a new bug for a build failure."""
   logs.info('Filing bug for new build failure (project=%s, build_type=%s, '
             'build_id=%s).' % (project_name, build_type, build_id))
@@ -147,8 +149,8 @@ def file_bug(issue_tracker, policy, project_name, build_id, ccs, build_type):
   issue.body = _get_issue_body(project_name, build_id, build_type)
   issue.status = policy.status('new')
 
-  for cc in ccs:
-    issue.ccs.add(cc)
+  if project_cc_group:
+    issue.ccs.add(project_cc_group)
 
   issue.save()
   return str(issue.id)
@@ -259,9 +261,11 @@ def _process_failures(projects, build_type):
               'Project %s is disabled, skipping bug filing.' % project_name)
           continue
 
+        project_cc_group = external_users.get_project_cc_group(
+            oss_fuzz_project.name)
         build_failure.issue_id = file_bug(issue_tracker, policy, project_name,
-                                          build['build_id'],
-                                          oss_fuzz_project.ccs, build_type)
+                                          build['build_id'], project_cc_group,
+                                          build_type)
       elif (build_failure.consecutive_failures -
             MIN_CONSECUTIVE_BUILD_FAILURES) % REMINDER_INTERVAL == 0:
         send_reminder(issue_tracker, build_failure.issue_id, build['build_id'])

src/clusterfuzz/_internal/cron/oss_fuzz_cc_groups.py

@@ -13,27 +13,36 @@
 # limitations under the License.
 """Cron to sync OSS-Fuzz projects groups used as CC in the issue tracker."""
 
+from clusterfuzz._internal.base import external_users
 from clusterfuzz._internal.base import utils
+from clusterfuzz._internal.config import local_config
 from clusterfuzz._internal.datastore import data_types
 from clusterfuzz._internal.datastore import ndb_utils
 from clusterfuzz._internal.google_cloud_utils import google_groups
 from clusterfuzz._internal.metrics import logs
 
-_CC_GROUP_SUFFIX = '-ccs@oss-fuzz.com'
-_CC_GROUP_DESC = 'External CCs in OSS-Fuzz issue tracker for project'
+
+def get_project_cc_group_description(project_name: str) -> str:
+  """Return issue CC group description for a oss-fuzz project."""
+  oss_fuzz_cc_desc = 'External CCs in OSS-Fuzz issue tracker for project'
+  return f'{oss_fuzz_cc_desc}: {project_name}'
 
 
 def sync_project_cc_group(project_name: str, ccs: list[str]):
-  """Sync the project's google group used for CCing in the issue tracker."""
-  group_name = f'{project_name}{_CC_GROUP_SUFFIX}'
+  """Sync the oss-fuzz project's group used for CCing in the issue tracker."""
+  group_name = external_users.get_project_cc_group(project_name)
 
   group_id = google_groups.get_group_id(group_name)
   # Create the group and bail out since the CIG API might delay to create a
   # new group. Add members will be done in the next cron run.
   if not group_id:
-    group_description = f'{_CC_GROUP_DESC}: {project_name}'
+    group_description = get_project_cc_group_description(project_name)
+    customer_id = local_config.ProjectConfig().get(
+        'issue_cc_groups.customer_id')
     created = google_groups.create_google_group(
-        group_name, group_description=group_description)
+        group_name,
+        group_description=group_description,
+        customer_id=customer_id)
     if not created:
       logs.warning('Failed to create or retrieve the issue tracker CC group '
                    f'for {project_name}')

src/clusterfuzz/_internal/cron/project_setup.py

@@ -23,6 +23,7 @@
 import requests
 import yaml
 
+from clusterfuzz._internal.base import feature_flags
 from clusterfuzz._internal.base import tasks
 from clusterfuzz._internal.base import untrusted
 from clusterfuzz._internal.base import utils
@@ -973,6 +974,11 @@ def _sync_job(self, project, info, corpus_bucket_name, quarantine_bucket_name,
   def sync_user_permissions(self, project, info):
     """Sync permissions of project based on project.yaml."""
     ccs = ccs_from_info(info)
+    fflag = feature_flags.FeatureFlags.UPDATE_OSS_FUZZ_USERS_AUTO_CC
+    update_cc_to_groups = fflag.enabled
+    auto_cc_type = (
+        data_types.AutoCCType.USE_CC_GROUP
+        if update_cc_to_groups else data_types.AutoCCType.ALL)
 
     for template in get_jobs_for_project(project, info):
       job_name = template.job_name(project, self._config_suffix)
@@ -997,14 +1003,18 @@ def sync_user_permissions(self, project, info):
 
         existing_permission = query.get()
         if existing_permission:
+          if (update_cc_to_groups and
+              existing_permission.auto_cc != auto_cc_type):
+            existing_permission.auto_cc = auto_cc_type
+            existing_permission.put()
           continue
-
+        # For OSS-Fuzz issue tracker, use the project cc google group.
         data_types.ExternalUserPermission(
             email=cc,
             entity_kind=data_types.PermissionEntityKind.JOB,
             entity_name=job_name,
             is_prefix=False,
-            auto_cc=data_types.AutoCCType.ALL).put()
+            auto_cc=auto_cc_type).put()
 
   def set_up(self, projects):
     """Do project setup. Return a list of all the project names that were set

src/clusterfuzz/_internal/datastore/data_types.py

@@ -208,6 +208,8 @@ class AutoCCType:
   ALL = 1
   # Auto-CC only for security issues.
   SECURITY = 2
+  # OSS-Fuzz specific - Auto-CC the user's project google group in all issues.
+  USE_CC_GROUP = 3
 
 
 # Type of permission. Used by ExternalUserPermision.

src/clusterfuzz/_internal/issue_management/issue_filer.py

@@ -315,7 +315,8 @@ def file_issue(testcase,
                issue_tracker,
                security_severity=None,
                user_email=None,
-               additional_ccs=None):
+               additional_ccs=None,
+               allow_project_cc_group=True):
   """File an issue for the given test case."""
   logs.info(f'Filing new issue for testcase: {testcase.key.id()}.')
 
@@ -406,8 +407,10 @@ def file_issue(testcase,
       testcase.overridden_fuzzer_name or testcase.fuzzer_name)
   ccs += external_users.cc_users_for_fuzzer(fully_qualified_fuzzer_name,
                                             testcase.security_flag)
-  ccs += external_users.cc_users_for_job(testcase.job_type,
-                                         testcase.security_flag)
+  ccs += external_users.cc_users_for_job(
+      testcase.job_type,
+      testcase.security_flag,
+      allow_cc_group=allow_project_cc_group)
 
   # Add the user as a cc if requested, and any default ccs for this job.
   # Check for additional ccs or labels from the job definition.

src/clusterfuzz/_internal/tests/appengine/handlers/cron/oss_fuzz_build_status_test.py

@@ -125,10 +125,13 @@ def setUp(self):
         ('policy_get',
          'clusterfuzz._internal.issue_management.issue_tracker_policy.get'),
         'clusterfuzz._internal.issue_management.issue_tracker_utils.get_issue_tracker',
-        'clusterfuzz._internal.metrics.logs.error',
-        'requests.get',
+        'clusterfuzz._internal.metrics.logs.error', 'requests.get',
+        'clusterfuzz._internal.config.local_config.ProjectConfig'
     ])
 
+    self.mock.ProjectConfig.return_value = {
+        'issue_cc_groups.domain': 'oss-fuzz.com'
+    }
     self.mock.utcnow.return_value = datetime.datetime(2018, 2, 1)
     self.mock.is_cron.return_value = True
 
@@ -374,7 +377,7 @@ def _mock_requests_get(url, timeout=None):
     data_types.OssFuzzProject(
         id='proj2', name='proj2', ccs=['a@user.com']).put()
     data_types.OssFuzzProject(
-        id='proj6', name='proj7', ccs=['b@user.com']).put()
+        id='proj6', name='proj6', ccs=['b@user.com']).put()
 
     oss_fuzz_build_status.main()
     self.assertCountEqual([
@@ -426,7 +429,7 @@ def _mock_requests_get(url, timeout=None):
 
     self.assertEqual(2, len(self.itm.issues))
     issue = self.itm.issues[1]
-    self.assertCountEqual(['a@user.com'], issue.cc)
+    self.assertCountEqual(['proj2-ccs@oss-fuzz.com'], issue.cc)
     self.assertEqual('New', issue.status)
     self.assertEqual('proj2: Fuzzing build failure', issue.summary)
     self.assertEqual(
@@ -445,7 +448,7 @@ def _mock_requests_get(url, timeout=None):
         'day once it is fixed.**', issue.body)
 
     issue = self.itm.issues[2]
-    self.assertCountEqual(['b@user.com'], issue.cc)
+    self.assertCountEqual(['proj6-ccs@oss-fuzz.com'], issue.cc)
     self.assertEqual('New', issue.status)
     self.assertEqual('proj6: Coverage build failure', issue.summary)
     self.assertEqual(

src/clusterfuzz/_internal/tests/appengine/handlers/cron/oss_fuzz_cc_groups_test.py

@@ -34,7 +34,12 @@ def setUp(self):
         'clusterfuzz._internal.google_cloud_utils.google_groups.delete_google_group_membership',
         'clusterfuzz._internal.google_cloud_utils.google_groups.set_oss_fuzz_access_settings',
         'clusterfuzz._internal.base.utils.is_service_account',
+        'clusterfuzz._internal.config.local_config.ProjectConfig'
     ])
+    self.mock.ProjectConfig.return_value = {
+        'issue_cc_groups.customer_id': 'C10101010',
+        'issue_cc_groups.domain': 'oss-fuzz.com'
+    }
 
   def test_main(self):
     """Test main execution for creating groups and syncing project ccs."""
@@ -62,7 +67,8 @@ def test_main(self):
     self.mock.create_google_group.assert_called_with(
         'project1-ccs@oss-fuzz.com',
         group_description=(
-            'External CCs in OSS-Fuzz issue tracker for project: project1'))
+            'External CCs in OSS-Fuzz issue tracker for project: project1'),
+        customer_id='C10101010')
 
     # project2 check
     self.mock.add_member_to_group.assert_called_with('group2_id',