MetaTL/trainer.py

from models import *
import os
import sys
import torch
import shutil
import logging
import numpy as np


class Trainer:
    def __init__(self, data_loaders, itemnum, parameter):
        self.parameter = parameter
        # data loader
        self.train_data_loader = data_loaders[0]
        self.dev_data_loader = data_loaders[1]
        self.test_data_loader = data_loaders[2]
        # parameters
        self.batch_size = parameter['batch_size']
        self.learning_rate = parameter['learning_rate']
        self.epoch = parameter['epoch']
        # self.print_epoch = parameter['print_epoch']
        # self.eval_epoch = parameter['eval_epoch']
        # self.device = torch.device(parameter['device'])
        self.device = parameter['device']

        self.MetaTL = MetaTL(itemnum, parameter)
        self.MetaTL.to(parameter['device'])

        self.optimizer = torch.optim.Adam(self.MetaTL.parameters(), self.learning_rate)

        if parameter['eval_epoch']:
            self.eval_epoch = parameter['eval_epoch']
        else:
            self.eval_epoch = 1000


    def rank_predict(self, data, x, ranks):
        # query_idx is the idx of positive score
        query_idx = x.shape[0] - 1
        # sort all scores with descending, because more plausible triple has higher score
        _, idx = torch.sort(x, descending=True)
        rank = list(idx.cpu().numpy()).index(query_idx) + 1
        ranks.append(rank)
        # update data
        if rank <= 10:
            data['Hits@10'] += 1
            data['NDCG@10'] += 1 / np.log2(rank + 1)
        if rank <= 5:
            data['Hits@5'] += 1
            data['NDCG@5'] += 1 / np.log2(rank + 1)
        if rank == 1:
            data['Hits@1'] += 1
            data['NDCG@1'] += 1 / np.log2(rank + 1)
        data['MRR'] += 1.0 / rank

    def do_one_step(self, task, iseval=False, curr_rel=''):
        loss, p_score, n_score = 0, 0, 0
        if not iseval:
            self.optimizer.zero_grad()
            p_score, n_score = self.MetaTL(task, iseval, curr_rel)
            y = torch.Tensor([1]).to(self.device)
            loss = self.MetaTL.loss_func(p_score, n_score,self.device)
            loss.backward()
            self.optimizer.step()
        elif curr_rel != '':
            p_score, n_score = self.MetaTL(task, iseval, curr_rel)
            y = torch.Tensor([1]).to(self.device)
            loss = self.MetaTL.loss_func(p_score, n_score,self.device)
        return loss, p_score, n_score

    def train(self):
        # initialization
        best_epoch = 0
        best_value = 0
        bad_counts = 0

        # training by epoch
        for e in range(self.epoch):
            # sample one batch from data_loader
            train_task, curr_rel = self.train_data_loader.next_batch()
            loss, _, _ = self.do_one_step(train_task, iseval=False, curr_rel=curr_rel)

            # do evaluation on specific epoch
            if e % self.eval_epoch == 0 and e != 0:
                loss_num = loss.detach().item()
                print("Epoch: {}\tLoss: {:.4f}".format(e, loss_num))

                print('Epoch  {} Validating...'.format(e))
                valid_data = self.eval(istest=False, epoch=e)

                print('Epoch  {} Testing...'.format(e))
                test_data = self.eval(istest=True, epoch=e)
                
        print('Finish')

    def eval(self, istest=False, epoch=None):
        # self.MetaTL.eval()
        
        self.MetaTL.rel_q_sharing = dict()

        if istest:
            data_loader = self.test_data_loader
        else:
            data_loader = self.dev_data_loader
        data_loader.curr_tri_idx = 0

        # initial return data of validation
        data = {'MRR': 0, 'Hits@1': 0, 'Hits@5': 0, 'Hits@10': 0, 'NDCG@1': 0, 'NDCG@5': 0, 'NDCG@10': 0}
        ranks = []

        t = 0
        temp = dict()
        total_loss = 0
        while True:
            # sample all the eval tasks
            eval_task, curr_rel = data_loader.next_one_on_eval()
            # at the end of sample tasks, a symbol 'EOT' will return
            if eval_task == 'EOT':
                break
            t += 1

            loss, p_score, n_score = self.do_one_step(eval_task, iseval=True, curr_rel=curr_rel)
            total_loss += loss

            x = torch.cat([n_score, p_score], 1).squeeze()

            self.rank_predict(data, x, ranks)

            # print current temp data dynamically
            for k in data.keys():
                temp[k] = data[k] / t
            # sys.stdout.write("{}\tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
            #     t, temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))
            # sys.stdout.flush()

        # print overall evaluation result and return it
        for k in data.keys():
            data[k] = round(data[k] / t, 3)

        
        if istest:
            print("TEST: \t test_loss: ",total_loss.detach().item())
            print("TEST: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']),"\n")
            with open('results.txt', 'a') as f:
                f.writelines("TEST: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r\n\n".format(
                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))
        else:
            print("VALID: \t validation_loss: ", total_loss.detach().item() )
            print("VALID: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))
            with open("results.txt",'a') as f:
                f.writelines("VALID: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))

        return data