Fix custom metric miscalculation (#307)

* Fix custom metric miscalculation

* add unit tests

Author: haixiw

src/sagemaker_xgboost_container/metrics/custom_metrics.py

@@ -239,8 +239,12 @@ def compute_multiclass_and_binary_metrics(metricfunc, preds, dtrain):
 
 
 def get_custom_metrics(eval_metrics):
-    """Get container defined metrics from metrics list."""
-    return set(eval_metrics).intersection(CUSTOM_METRICS.keys())
+    """Get container defined metrics from metrics list.
+
+    The order of the returning custom metrics need to be consistent with the input for distributed training.
+    Otherwise, metrics reported from each host will be miscalculated in the master host. (P70679777)
+    """
+    return [eval_m for eval_m in eval_metrics if eval_m in CUSTOM_METRICS.keys()]
 
 
 def configure_feval(custom_metric_list):

test/unit/algorithm_mode/test_custom_metrics.py

@@ -13,9 +13,10 @@
 import numpy as np
 import xgboost as xgb
 from math import log, sqrt
+import unittest
 from sagemaker_xgboost_container.metrics.custom_metrics import accuracy, f1, mse, r2, f1_binary, f1_macro, \
     precision_macro, precision_micro, recall_macro, recall_micro, mae, rmse, balanced_accuracy, \
-    precision, recall
+    precision, recall, get_custom_metrics
 
 
 binary_train_data = np.random.rand(10, 2)
@@ -190,3 +191,10 @@ def test_mae():
     mae_score_name, mae_score_result = mae(regression_preds, regression_dtrain)
     assert mae_score_name == 'mae'
     assert mae_score_result == .5
+
+
+class TestCustomMetric(unittest.TestCase):
+    def test_get_custom_metrics(self):
+        eval_metrics = ["mse", "rmse", "mae", "r2", "wrong_metric"]
+        res = get_custom_metrics(eval_metrics)
+        self.assertListEqual(res, ["mse", "rmse", "mae", "r2"])