filter_data.py

import pandas as pd
from collections import Counter

inter_file_path = './uk_session_train.txt'
session_min_inter = 2
item_min_inter = 5
filter_file_path = r'./uk_session_filter_session_'+str(session_min_inter)+'_item_'+str(item_min_inter)+'.txt'


session_list = []
with open(inter_file_path,'r') as f:
    for idx,session in enumerate(f.readlines()):
        tmp = session.strip().split(' ')
        session_list.append(tmp)
print('all session num '+len(session_list))

def drop_item(sessions,limit=item_min_inter):
    aaa = {}
    for i in sessions:
        for j in i:
            if j in aaa:
                aaa[j] +=1
            else:
                aaa[j] = 1

    filter_aaa = {}
    for i in aaa:
        if aaa[i]>=limit:
            filter_aaa[i]=aaa[i]
    return filter_aaa

def drop_session(sessions,aaa,limit = session_min_inter):
    new_sessions = []
    new_items = set(aaa.keys())
    for session in sessions:
        tmp = list(filter(lambda x:x in new_items,session))
        if len(tmp)>=limit:
            new_sessions.append(tmp)
    return new_sessions

# sessions = [['1','2','3'],['2','4'],['1','2','3'],['1','2','3'],['1','2','3']]
# aaa = {'1':10,'2':3,'3':6,'4':1}
if __name__ == '__main__':
    while True:
        flag = session_list
        tmp = drop_item(session_list)
        # print(aaa)
        session_list = drop_session(session_list,tmp)
        if session_list == flag:
            break
    print('all seesion num (after filtering): '+ str(len(session_list)))
    print('min seesion len: '+str(min(map(lambda x: len(x), session_list))))
    final_item = {}
    for sessions in session_list:
        for item in sessions:
            if item in final_item:
                final_item[item]+=1
            else:
                final_item[item]=1
    print('all item num: '+ str(len(final_item)))
    print('min item freq:'+ str(min(final_item.values())))