Add files via upload

Author: fireindark707

cal_column_similarity.py

@@ -1,4 +1,5 @@
 from relation_features import make_data_from
+from utils import make_csv_from_json
 from train import test
 import numpy as np
 import pandas as pd
@@ -16,6 +17,7 @@ def create_similarity_matrix(pth,preds,pred_labels_list,strategy="one-to-many"):
     """
     Create a similarity matrix from the prediction
     """
+    predicted_pairs = []
     preds = np.array(preds)
     preds = np.mean(preds,axis=0)
     pred_labels_list = np.array(pred_labels_list)
@@ -28,6 +30,7 @@ def create_similarity_matrix(pth,preds,pred_labels_list,strategy="one-to-many"):
     df2_cols = df2.columns
     # create similarity matrix for pred values 
     preds_matrix = np.array(preds).reshape(len(df1_cols),len(df2_cols))
+    # create similarity matrix for pred labels
     if strategy == "one-to-many":
         pred_labels_matrix = np.array(pred_labels).reshape(len(df1_cols),len(df2_cols))
     elif strategy == "one-to-one":
@@ -41,11 +44,19 @@ def create_similarity_matrix(pth,preds,pred_labels_list,strategy="one-to-many"):
                         pred_labels_matrix[i,j] = 1
     df_pred = pd.DataFrame(preds_matrix,columns=df2_cols,index=df1_cols)
     df_pred_labels = pd.DataFrame(pred_labels_matrix,columns=df2_cols,index=df1_cols)
-    return df_pred,df_pred_labels
+    for i in range(len(df_pred_labels)):
+        for j in range(len(df_pred_labels.iloc[i])):
+            if df_pred_labels.iloc[i,j] == 1:
+                predicted_pairs.append((df_pred.index[i],df_pred.columns[j],df_pred.iloc[i,j]))
+    return df_pred,df_pred_labels,predicted_pairs
 
 if __name__ == '__main__':
     pth = args.path
     model_pth = args.model
+    # transform jsonl or json file to csv
+    for file in os.listdir(args.path):
+        if file.endswith('.json') or file.endswith('.jsonl'):
+            make_csv_from_json(pth+"/"+file)
 
     features,_ = make_data_from(pth,"test")
     preds = []
@@ -64,6 +75,9 @@ def create_similarity_matrix(pth,preds,pred_labels_list,strategy="one-to-many"):
         pred_labels_list.append(pred_labels)
         del bst
 
-    df_pred,df_pred_labels = create_similarity_matrix(pth,preds,pred_labels_list,strategy=args.strategy)
+    df_pred,df_pred_labels,predicted_pairs = create_similarity_matrix(pth,preds,pred_labels_list,strategy=args.strategy)
     df_pred.to_csv(pth+"/similarity_matrix_value.csv",index=True)
-    df_pred_labels.to_csv(pth+"/similarity_matrix_label.csv",index=True)
+    df_pred_labels.to_csv(pth+"/similarity_matrix_label.csv",index=True)
+
+    for pair_tuple in predicted_pairs:
+        print(pair_tuple)

utils.py

@@ -0,0 +1,48 @@
+import pandas as pd
+import json
+from collections import defaultdict
+import re
+
+def find_all_keys_values(json_data,parent_key):
+    """
+    Find all keys that don't have list or dictionary values and their values. Key should be saved with its parent key like "parent-key.key".
+    """
+    key_values = defaultdict(list)
+    for key, value in json_data.items():
+        if isinstance(value, dict):
+            child_key_values = find_all_keys_values(value,key)
+            for child_key, child_value in child_key_values.items():
+                key_values[child_key].extend(child_value)
+        elif isinstance(value, list):
+            for item in value:
+                if isinstance(item, dict):
+                    child_key_values = find_all_keys_values(item,key)
+                    for child_key, child_value in child_key_values.items():
+                        key_values[child_key].extend(child_value)
+                else:
+                    key_values[parent_key+"."+key].append(item)
+        else:
+            key_values[parent_key+"."+key].append(value)
+    return key_values
+
+def make_csv_from_json(file_path):
+    """
+    Make csv file from json file.
+    """
+    with open(file_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # find key_values
+    if isinstance(data, dict):
+        key_values = find_all_keys_values(data,"")
+    elif isinstance(data, list):
+        key_values = find_all_keys_values({"data":data},"")
+    else:
+        raise ValueError('Your input JsonData is not a dictionary or list')
+
+    key_values = {k:v for k,v in key_values.items() if len(v)>1}
+
+    df = pd.DataFrame({k:pd.Series(v) for k,v in key_values.items()})
+    # save to csv
+    save_pth = re.sub(r'\.jsonl?','.csv',file_path)
+    df.to_csv(save_pth, index=False, encoding='utf-8')