data_loader.py

# -*- coding: utf-8 -*-
import numpy as np
from feature_maps import USER_ACTIVE_DEGREE_MAP, RANGE_MAP, VIDEO_TYPE_MAP


def parse_line(line, seq_len):
    parts = line.strip().split('\t')
    if len(parts) != 23:
        raise ValueError("Bad line format")

    label = float(parts[0])

    user_active_degree = USER_ACTIVE_DEGREE_MAP.get(parts[1], 0)
    is_lowactive_period = int(parts[2])
    is_live_streamer = int(parts[3])
    is_video_author = int(parts[4])

    follow_user_num_range = RANGE_MAP.get(parts[5], 0)
    fans_user_num_range = RANGE_MAP.get(parts[6], 0)
    register_days_range = RANGE_MAP.get(parts[7], 0)

    onehot_feat1 = int(parts[8])
    onehot_feat2 = int(parts[9])
    onehot_feat3 = int(parts[10])

    target_video_id = int(parts[11])
    target_author_id = int(parts[12])
    target_video_type = VIDEO_TYPE_MAP.get(parts[13], 0)
    target_music_type = int(parts[14])

    hist_video_ids = [int(x) for x in parts[15].split(',')] if parts[15] else []
    hist_hour_0_23 = [float(x) for x in parts[16].split(',')] if parts[16] else []
    hist_weekday_0_6 = [float(x) for x in parts[17].split(',')] if parts[17] else []
    hist_delta_t_hours = [float(x) for x in parts[18].split(',')] if parts[18] else []
    hist_time_seconds = [float(x) for x in parts[19].split(',')] if parts[19] else []
    hist_prev_day_segment_stat = [float(x) for x in parts[20].split(',')] if parts[20] else []
    hist_30_segment_avg_stat = [float(x) for x in parts[21].split(',')] if parts[21] else []

    hist_len = min(int(parts[22]), seq_len)

    hist_video_ids = hist_video_ids[:seq_len]
    hist_hour_0_23 = hist_hour_0_23[:seq_len]
    hist_weekday_0_6 = hist_weekday_0_6[:seq_len]
    hist_delta_t_hours = hist_delta_t_hours[:seq_len]
    hist_time_seconds = hist_time_seconds[:seq_len]
    hist_prev_day_segment_stat = hist_prev_day_segment_stat[:seq_len]
    hist_30_segment_avg_stat = hist_30_segment_avg_stat[:seq_len]

    pad_len = seq_len - hist_len

    hist_video_ids += [0] * pad_len
    hist_hour_0_23 += [0.0] * pad_len
    hist_weekday_0_6 += [0.0] * pad_len
    hist_delta_t_hours += [0.0] * pad_len
    hist_time_seconds += [0.0] * pad_len
    hist_prev_day_segment_stat += [0.0] * pad_len
    hist_30_segment_avg_stat += [0.0] * pad_len

    seq_mask = [True] * hist_len + [False] * pad_len

    return {
        "labels": [label],
        "user_active_degree": user_active_degree,
        "is_lowactive_period": is_lowactive_period,
        "is_live_streamer": is_live_streamer,
        "is_video_author": is_video_author,
        "follow_user_num_range": follow_user_num_range,
        "fans_user_num_range": fans_user_num_range,
        "register_days_range": register_days_range,
        "onehot_feat1": onehot_feat1,
        "onehot_feat2": onehot_feat2,
        "onehot_feat3": onehot_feat3,
        "target_video_id": target_video_id,
        "target_author_id": target_author_id,
        "target_video_type": target_video_type,
        "target_music_type": target_music_type,
        "item_ids": hist_video_ids,
        "seq_mask": seq_mask,
        "hour_0_23": hist_hour_0_23,
        "weekday_0_6": hist_weekday_0_6,
        "delta_t_hours": hist_delta_t_hours,
        "seq_time_seconds": hist_time_seconds,
        "prev_day_segment_stat": hist_prev_day_segment_stat,
        "hist30_segment_avg_stat": hist_30_segment_avg_stat
    }


def load_formatted_batch_from_txt(file_obj, batch_size, seq_len):
    batch = {
        "labels": [],
        "user_active_degree": [],
        "is_lowactive_period": [],
        "is_live_streamer": [],
        "is_video_author": [],
        "follow_user_num_range": [],
        "fans_user_num_range": [],
        "register_days_range": [],
        "onehot_feat1": [],
        "onehot_feat2": [],
        "onehot_feat3": [],
        "target_video_id": [],
        "target_author_id": [],
        "target_video_type": [],
        "target_music_type": [],
        "item_ids": [],
        "seq_mask": [],
        "hour_0_23": [],
        "weekday_0_6": [],
        "delta_t_hours": [],
        "seq_time_seconds": [],
        "prev_day_segment_stat": [],
        "hist30_segment_avg_stat": []
    }

    cnt = 0
    while cnt < batch_size:
        line = file_obj.readline()
        if not line:
            break
        line = line.strip()
        if not line:
            continue

        sample = parse_line(line, seq_len)
        for k in batch:
            batch[k].append(sample[k])
        cnt += 1

    if cnt == 0:
        return None

    return {
        "labels": np.array(batch["labels"], dtype=np.float32),
        "user_active_degree": np.array(batch["user_active_degree"], dtype=np.int32),
        "is_lowactive_period": np.array(batch["is_lowactive_period"], dtype=np.int32),
        "is_live_streamer": np.array(batch["is_live_streamer"], dtype=np.int32),
        "is_video_author": np.array(batch["is_video_author"], dtype=np.int32),
        "follow_user_num_range": np.array(batch["follow_user_num_range"], dtype=np.int32),
        "fans_user_num_range": np.array(batch["fans_user_num_range"], dtype=np.int32),
        "register_days_range": np.array(batch["register_days_range"], dtype=np.int32),
        "onehot_feat1": np.array(batch["onehot_feat1"], dtype=np.int32),
        "onehot_feat2": np.array(batch["onehot_feat2"], dtype=np.int32),
        "onehot_feat3": np.array(batch["onehot_feat3"], dtype=np.int32),
        "target_video_id": np.array(batch["target_video_id"], dtype=np.int32),
        "target_author_id": np.array(batch["target_author_id"], dtype=np.int32),
        "target_video_type": np.array(batch["target_video_type"], dtype=np.int32),
        "target_music_type": np.array(batch["target_music_type"], dtype=np.int32),
        "item_ids": np.array(batch["item_ids"], dtype=np.int32),
        "seq_mask": np.array(batch["seq_mask"], dtype=np.bool_),
        "hour_0_23": np.array(batch["hour_0_23"], dtype=np.float32),
        "weekday_0_6": np.array(batch["weekday_0_6"], dtype=np.float32),
        "delta_t_hours": np.array(batch["delta_t_hours"], dtype=np.float32),
        "seq_time_seconds": np.array(batch["seq_time_seconds"], dtype=np.float32),
        "prev_day_segment_stat": np.array(batch["prev_day_segment_stat"], dtype=np.float32),
        "hist30_segment_avg_stat": np.array(batch["hist30_segment_avg_stat"], dtype=np.float32)
    }