Merge pull request #1092 from ViktorTigerstrom/2025-06-migrate-privacy-mapper

[sql-44] firewalldb: add migration code for privacy mapper from kvdb to SQL

Author: ellemouton

firewalldb/sql_migration.go

@@ -67,14 +67,16 @@ func (e *kvEntry) namespacedKey() string {
 	return ns
 }
 
+// privacyPairs is a type alias for a map that holds the privacy pairs, where
+// the outer key is the group ID, and the value is a map of real to pseudo
+// values.
+type privacyPairs = map[int64]map[string]string
+
 // MigrateFirewallDBToSQL runs the migration of the firwalldb stores from the
 // bbolt database to a SQL database. The migration is done in a single
 // transaction to ensure that all rows in the stores are migrated or none at
 // all.
 //
-// Note that this migration currently only migrates the kvstores, but will be
-// extended in the future to also migrate the privacy mapper and action stores.
-//
 // NOTE: As sessions may contain linked sessions and accounts, the sessions and
 // accounts sql migration MUST be run prior to this migration.
 func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB,
@@ -87,10 +89,14 @@ func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 		return err
 	}
 
+	err = migratePrivacyMapperDBToSQL(ctx, kvStore, sqlTx)
+	if err != nil {
+		return err
+	}
+
 	log.Infof("The rules DB has been migrated from KV to SQL.")
 
-	// TODO(viktor): Add migration for the privacy mapper and the action
-	// stores.
+	// TODO(viktor): Add migration for the action stores.
 
 	return nil
 }
@@ -490,3 +496,281 @@ func verifyBktKeys(bkt *bbolt.Bucket, errorOnKeyValues bool,
 		return fmt.Errorf("unexpected key found: %s", key)
 	})
 }
+
+// migratePrivacyMapperDBToSQL runs the migration of the privacy mapper store
+// from the KV database to the SQL database. The function also asserts that the
+// migrated values match the original values in the privacy mapper store.
+func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) error {
+
+	log.Infof("Starting migration of the privacy mapper store to SQL")
+
+	// 1) Collect all privacy pairs from the KV store.
+	privPairs, err := collectPrivacyPairs(ctx, kvStore, sqlTx)
+	if err != nil {
+		return fmt.Errorf("error migrating privacy mapper store: %w",
+			err)
+	}
+
+	// 2) Insert all collected privacy pairs into the SQL database.
+	err = insertPrivacyPairs(ctx, sqlTx, privPairs)
+	if err != nil {
+		return fmt.Errorf("insertion of privacy pairs failed: %w", err)
+	}
+
+	// 3) Validate that all inserted privacy pairs match the original values
+	// in the KV store. Note that this is done after all values have been
+	// inserted, to ensure that the migration doesn't overwrite any values
+	// after they were inserted.
+	err = validatePrivacyPairsMigration(ctx, sqlTx, privPairs)
+	if err != nil {
+		return fmt.Errorf("migration validation of privacy pairs "+
+			"failed: %w", err)
+	}
+
+	log.Infof("Migration of the privacy mapper stores to SQL completed. "+
+		"Total number of rows migrated: %d", len(privPairs))
+
+	return nil
+}
+
+// collectPrivacyPairs collects all privacy pairs from the KV store.
+func collectPrivacyPairs(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) (privacyPairs, error) {
+
+	groupPairs := make(privacyPairs)
+
+	return groupPairs, kvStore.View(func(kvTx *bbolt.Tx) error {
+		bkt := kvTx.Bucket(privacyBucketKey)
+		if bkt == nil {
+			// If we haven't generated any privacy bucket yet,
+			// we can skip the migration, as there are no privacy
+			// pairs to migrate.
+			return nil
+		}
+
+		return bkt.ForEach(func(groupId, v []byte) error {
+			if v != nil {
+				return fmt.Errorf("expected only buckets "+
+					"under %s bkt, but found value %s",
+					privacyBucketKey, v)
+			}
+
+			gBkt := bkt.Bucket(groupId)
+			if gBkt == nil {
+				return fmt.Errorf("group bkt for group id "+
+					"%s not found", groupId)
+			}
+
+			groupSqlId, err := sqlTx.GetSessionIDByAlias(
+				ctx, groupId,
+			)
+			if errors.Is(err, sql.ErrNoRows) {
+				return fmt.Errorf("session with group id %x "+
+					"not found in sql db", groupId)
+			} else if err != nil {
+				return err
+			}
+
+			groupRealToPseudoPairs, err := collectGroupPairs(gBkt)
+			if err != nil {
+				return fmt.Errorf("processing group bkt "+
+					"for group id %s (sqlID %d) failed: %w",
+					groupId, groupSqlId, err)
+			}
+
+			groupPairs[groupSqlId] = groupRealToPseudoPairs
+
+			return nil
+		})
+	})
+}
+
+// collectGroupPairs collects all privacy pairs for a specific session group,
+// i.e. the group buckets under the privacy mapper bucket in the KV store.
+// The function returns them as a map, where the key is the real value, and
+// the value for the key is the pseudo values.
+// It also checks that the pairs are consistent, i.e. that for each real value
+// there is a corresponding pseudo value, and vice versa. If the pairs are
+// inconsistent, it returns an error indicating the mismatch.
+func collectGroupPairs(bkt *bbolt.Bucket) (map[string]string, error) {
+	var (
+		realToPseudoRes map[string]string
+		pseudoToRealRes map[string]string
+		err             error
+	)
+
+	if realBkt := bkt.Bucket(realToPseudoKey); realBkt != nil {
+		realToPseudoRes, err = collectPairs(realBkt)
+		if err != nil {
+			return nil, fmt.Errorf("fetching real to pseudo pairs "+
+				"failed: %w", err)
+		}
+	} else {
+		return nil, fmt.Errorf("%s bucket not found", realToPseudoKey)
+	}
+
+	if pseudoBkt := bkt.Bucket(pseudoToRealKey); pseudoBkt != nil {
+		pseudoToRealRes, err = collectPairs(pseudoBkt)
+		if err != nil {
+			return nil, fmt.Errorf("fetching pseudo to real pairs "+
+				"failed: %w", err)
+		}
+	} else {
+		return nil, fmt.Errorf("%s bucket not found", pseudoToRealKey)
+	}
+
+	if len(realToPseudoRes) != len(pseudoToRealRes) {
+		return nil, fmt.Errorf("missmatch between nubmer of pairs in "+
+			"%s bucket (pairs found: %d) and %s bucket (pairs "+
+			"found: %d)", realToPseudoKey, len(realToPseudoRes),
+			pseudoToRealKey, len(pseudoToRealRes))
+	}
+
+	for realVal, pseudoVal := range realToPseudoRes {
+		if rv, ok := pseudoToRealRes[pseudoVal]; !ok || rv != realVal {
+			return nil, fmt.Errorf("the real value %s found in "+
+				"the %s bucket doesn't match the value %s "+
+				"found in the %s bucket",
+				realVal, realToPseudoKey, rv, pseudoToRealKey)
+		}
+	}
+
+	return realToPseudoRes, nil
+}
+
+// collectPairs collects all privacy pairs from a specific realToPseudoKey or
+// pseudoToRealKey bucket in the KV store. It returns a map where the key is
+// the real value or pseudo value, and the value is the corresponding pseudo
+// value or real value, respectively (depending on if the realToPseudo or
+// pseudoToReal bucket is passed to the function).
+func collectPairs(pairsBucket *bbolt.Bucket) (map[string]string, error) {
+	pairsRes := make(map[string]string)
+
+	return pairsRes, pairsBucket.ForEach(func(k, v []byte) error {
+		if v == nil {
+			return fmt.Errorf("expected only key-values under "+
+				"pairs bucket, but found bucket %s", k)
+		}
+
+		if len(v) == 0 {
+			return fmt.Errorf("empty value stored for privacy "+
+				"pairs key %s", k)
+		}
+
+		pairsRes[string(k)] = string(v)
+
+		return nil
+	})
+}
+
+// insertPrivacyPairs inserts the collected privacy pairs into the SQL database.
+func insertPrivacyPairs(ctx context.Context, sqlTx SQLQueries,
+	pairs privacyPairs) error {
+
+	for groupId, groupPairs := range pairs {
+		err := insertGroupPairs(ctx, sqlTx, groupId, groupPairs)
+		if err != nil {
+			return fmt.Errorf("inserting group pairs for group "+
+				"id %d failed: %w", groupId, err)
+		}
+	}
+
+	return nil
+}
+
+// insertGroupPairs inserts the privacy pairs for a specific group into
+// the SQL database. It checks for duplicates before inserting, and returns
+// an error if a duplicate pair is found. The function takes a map of real
+// to pseudo values, where the key is the real value and the value is the
+// corresponding pseudo value.
+func insertGroupPairs(ctx context.Context, sqlTx SQLQueries, groupID int64,
+	pairs map[string]string) error {
+
+	for realVal, pseudoVal := range pairs {
+		err := sqlTx.InsertPrivacyPair(
+			ctx, sqlc.InsertPrivacyPairParams{
+				GroupID:   groupID,
+				RealVal:   realVal,
+				PseudoVal: pseudoVal,
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("inserting privacy pair %s:%s "+
+				"failed: %w", realVal, pseudoVal, err)
+		}
+	}
+
+	return nil
+}
+
+// validatePrivacyPairsMigration validates that the migrated privacy pairs
+// match the original values in the KV store.
+func validatePrivacyPairsMigration(ctx context.Context, sqlTx SQLQueries,
+	pairs privacyPairs) error {
+
+	for groupId, groupPairs := range pairs {
+		err := validateGroupPairsMigration(
+			ctx, sqlTx, groupId, groupPairs,
+		)
+		if err != nil {
+			return fmt.Errorf("migration validation of privacy "+
+				"pairs for group %d failed: %w", groupId, err)
+		}
+	}
+
+	return nil
+}
+
+// validateGroupPairsMigration validates that the migrated privacy pairs for
+// a specific group match the original values in the KV store. It checks that
+// for each real value, the pseudo value in the SQL database matches the
+// original pseudo value, and vice versa. If any mismatch is found, it returns
+// an error indicating the mismatch.
+func validateGroupPairsMigration(ctx context.Context, sqlTx SQLQueries,
+	groupID int64, pairs map[string]string) error {
+
+	for realVal, pseudoVal := range pairs {
+		resPseudoVal, err := sqlTx.GetPseudoForReal(
+			ctx, sqlc.GetPseudoForRealParams{
+				GroupID: groupID,
+				RealVal: realVal,
+			},
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("migrated privacy pair %s:%s not "+
+				"found for real value", realVal, pseudoVal)
+		}
+		if err != nil {
+			return err
+		}
+
+		if resPseudoVal != pseudoVal {
+			return fmt.Errorf("pseudo value in db %s, does not "+
+				"match original value %s, for real value %s",
+				resPseudoVal, pseudoVal, realVal)
+		}
+
+		resRealVal, err := sqlTx.GetRealForPseudo(
+			ctx, sqlc.GetRealForPseudoParams{
+				GroupID:   groupID,
+				PseudoVal: pseudoVal,
+			},
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("migrated privacy pair %s:%s not "+
+				"found for pseudo value", realVal, pseudoVal)
+		}
+		if err != nil {
+			return err
+		}
+
+		if resRealVal != realVal {
+			return fmt.Errorf("real value in db %s, does not "+
+				"match original value %s, for pseudo value %s",
+				resRealVal, realVal, pseudoVal)
+		}
+	}
+
+	return nil
+}