Implementing Robustify Negative sampling and undo prepossessing

main.py

 from utils import *
 from sampler import *
 import json
+import torch
 
 import argparse
 
 def get_params():
     args = argparse.ArgumentParser()
     args.add_argument("-data", "--dataset", default="electronics", type=str)
-    args.add_argument("-seed", "--seed", default=None, type=int)
+    args.add_argument("-seed", "--seed", default=7, type=int)
     args.add_argument("-K", "--K", default=3, type=int) #NUMBER OF SHOT
 
-    args.add_argument("-dim", "--embed_dim", default=100, type=int)
+    args.add_argument("-dim", "--embed_dim", default=128, type=int)
     args.add_argument("-bs", "--batch_size", default=1024, type=int)
     args.add_argument("-lr", "--learning_rate", default=0.001, type=float)
 
     args.add_argument("-epo", "--epoch", default=100000, type=int)
     args.add_argument("-prt_epo", "--print_epoch", default=100, type=int)
-    args.add_argument("-eval_epo", "--eval_epoch", default=1000, type=int)
+    args.add_argument("-eval_epo", "--eval_epoch", default=500, type=int)
 
     args.add_argument("-b", "--beta", default=5, type=float)
     args.add_argument("-m", "--margin", default=1, type=float)
     args.add_argument("-p", "--dropout_p", default=0.5, type=float)
 
     args.add_argument("-gpu", "--device", default=0, type=int)
-    args.add_argument("--number_of_neg",default=5,type=int)
+    args.add_argument("--number_of_neg",default=1,type=int)
 
 
 
         params[k] = v
 
 
-
+    params['device'] = torch.device('cuda:'+str(args.device))
-    # params['device'] = torch.device('cuda:'+str(args.device))
+    # params['device'] = args.device
-    params['device'] = args.device
     # params['device'] = torch.device('cpu')
 
     return params, args
 
 if __name__ == '__main__':
+    print(torch.__version__)
     print(torch.cuda.is_available())
     params, args = get_params()
 
+    params['varset_size'] = 1000
+    params['alpha'] = 0.5
+    params['S1'] = 40
+    params['S2_div_S1'] = 1
+    params['temperature'] = 1.0
+    params['warmup'] = 20.0
+
     if params['seed'] is not None:
         SEED = params['seed']
         torch.manual_seed(SEED)
     print("===============", torch.cuda.device_count(), "=======")
     user_train, usernum_train, itemnum, user_input_test, user_test, user_input_valid, user_valid = data_load(args.dataset, args.K)    
 
-    sampler = WarpSampler(user_train, usernum_train, itemnum, batch_size=args.batch_size, maxlen=args.K, n_workers=3,params=params)
+    sampler = WarpSampler(user_train, usernum_train, itemnum, batch_size=args.batch_size, maxlen=args.K, n_workers=3)
 
     sampler_test = DataLoader(user_input_test, user_test, itemnum, params)
 
     sampler_valid = DataLoader(user_input_valid, user_valid, itemnum, params)
 
     print("===============", torch.cuda.device_count(), "=======")
-
+    trainer = Trainer([sampler, sampler_valid, sampler_test], itemnum, params,usernum_train,user_train)
-    trainer = Trainer([sampler, sampler_valid, sampler_test], itemnum, params)
     print("===============", torch.cuda.device_count(), "=======")
     trainer.train()
 

models.py

 import torch
 import torch.nn as nn
 from torch.nn import functional as F
+from numpy import linalg as LA
+import numpy as np
+
 
 class Embedding(nn.Module):
     def __init__(self, num_ent, parameter):
         # self.device = torch.device('cuda:0')
         self.device = torch.device(parameter['device'])
         self.es = parameter['embed_dim']
-        
+
         self.embedding = nn.Embedding(num_ent + 1, self.es)
         nn.init.xavier_uniform_(self.embedding.weight)
 
         idx = torch.LongTensor(idx).to(self.device)
         return self.embedding(idx)
 
+
 class MetaLearner(nn.Module):
     def __init__(self, K, embed_size=100, num_hidden1=500, num_hidden2=200, out_size=100, dropout_p=0.5):
         super(MetaLearner, self).__init__()
         self.out_size = embed_size
         self.hidden_size = embed_size
         # self.rnn = nn.LSTM(embed_size,self.hidden_size,2,dropout=0.2)
-        self.rnn = nn.GRU(input_size=embed_size,hidden_size=self.hidden_size, num_layers=1)
+        self.rnn = nn.GRU(input_size=embed_size, hidden_size=self.embed_size * 2, num_layers=1)
+        self.activation = nn.LeakyReLU()
+        self.linear = nn.Linear(self.embed_size * 2, self.embed_size)
+        self.norm = nn.BatchNorm1d(num_features=self.out_size)
 
-        # nn.init.xavier_normal_(self.rnn.all_weights)
+        # nn.init.xavier_normal_(self.linear.weight)
 
-    def forward(self, inputs):
+    def forward(self, inputs, evaluation=False):
         size = inputs.shape
-        x = torch.stack([inputs[:,0,0,:],inputs[:,0,1,:],inputs[:,1,1,:]],dim=1)
+        x = torch.stack([inputs[:, 0, 0, :], inputs[:, 0, 1, :], inputs[:, 1, 1, :]], dim=1)
-        x = x.transpose(0,1)
+        x = x.transpose(0, 1)
 
         # _,(x,c) = self.rnn(x)
-        x,c = self.rnn(x)
+        x, c = self.rnn(x)
         x = x[-1]
-        x = x.squeeze(0)
+        if not evaluation:
+            x = x.squeeze(0)
+
+        x = self.activation(x)
+        x = self.linear(x)
+        x = self.norm(x)
 
         return x.view(size[0], 1, 1, self.out_size)
 
+
 class EmbeddingLearner(nn.Module):
     def __init__(self):
         super(EmbeddingLearner, self).__init__()
         n_score = score[:, pos_num:]
         return p_score, n_score
 
-def bpr_loss(p_scores, n_values,device):
+def bpr_loss(p_scores, n_values, device):
     ratio = int(n_values.shape[1] / p_scores.shape[1])
-    temp_pvalues = torch.tensor([],device=device)
+    temp_pvalues = torch.tensor([], device=device)
     for i in range(p_scores.shape[1]):
         temp_pvalues = torch.cat((temp_pvalues, p_scores[:, i, None].expand(-1, ratio)), dim=1)
 
-    d = torch.sub(temp_pvalues,n_values)
+    d = torch.sub(temp_pvalues, n_values)
     t = F.logsigmoid(d)
-    loss = -1 * (1.0/n_values.shape[1]) * t.sum(dim=1)
+    loss = -1 * (1.0 / n_values.shape[1]) * t.sum(dim=1)
     loss = loss.sum(dim=0)
     return loss
 
-def bpr_max_loss(p_scores, n_values,device):
+def bpr_max_loss(p_scores, n_values, device):
-    s = F.softmax(n_values,dim=1)
+    s = F.softmax(n_values, dim=1)
     ratio = int(n_values.shape[1] / p_scores.shape[1])
-    temp_pvalues = torch.tensor([],device=device)
+    temp_pvalues = torch.tensor([], device=device)
     for i in range(p_scores.shape[1]):
-        temp_pvalues = torch.cat((temp_pvalues,p_scores[:,i,None].expand(-1,ratio)),dim=1)
+        temp_pvalues = torch.cat((temp_pvalues, p_scores[:, i, None].expand(-1, ratio)), dim=1)
 
-    d = torch.sigmoid(torch.sub(temp_pvalues,n_values))
+    d = torch.sigmoid(torch.sub(temp_pvalues, n_values))
-    t = torch.mul(s,d)
+    t = torch.mul(s, d)
     loss = -1 * torch.log(t.sum(dim=1))
     loss = loss.sum()
     return loss
 
-def bpr_max_loss_regularized(p_scores, n_values,device,l=0.0001):
+def bpr_max_loss_regularized(p_scores, n_values, device, l=0.0001):
-    s = F.softmax(n_values,dim=1)
+    s = F.softmax(n_values, dim=1)
     ratio = int(n_values.shape[1] / p_scores.shape[1])
-    temp_pvalues = torch.tensor([],device=device)
+    temp_pvalues = torch.tensor([], device=device)
     for i in range(p_scores.shape[1]):
-        temp_pvalues = torch.cat((temp_pvalues,p_scores[:,i,None].expand(-1,ratio)),dim=1)
+        temp_pvalues = torch.cat((temp_pvalues, p_scores[:, i, None].expand(-1, ratio)), dim=1)
 
-    d = torch.sigmoid(torch.sub(temp_pvalues,n_values))
+    d = torch.sigmoid(torch.sub(temp_pvalues, n_values))
-    t = torch.mul(s,d)
+    t = torch.mul(s, d)
     loss = -1 * torch.log(t.sum(dim=1))
     loss = loss.sum()
 
-    loss2 = torch.mul(s,n_values**2)
+    loss2 = torch.mul(s, n_values ** 2)
     loss2 = loss2.sum(dim=1)
     loss2 = loss2.sum()
-    return loss + l*loss2
+    return loss + l * loss2
 
-def top_loss(p_scores, n_values,device):
+def top_loss(p_scores, n_values, device):
     ratio = int(n_values.shape[1] / p_scores.shape[1])
-    temp_pvalues = torch.tensor([],device=device)
+    temp_pvalues = torch.tensor([], device=device)
     for i in range(p_scores.shape[1]):
         temp_pvalues = torch.cat((temp_pvalues, p_scores[:, i, None].expand(-1, ratio)), dim=1)
 
-    t1 = torch.sigmoid(torch.sub(n_values , temp_pvalues))
+    t1 = torch.sigmoid(torch.sub(n_values, temp_pvalues))
-    t2 = torch.sigmoid(torch.pow(n_values,2))
+    t2 = torch.sigmoid(torch.pow(n_values, 2))
-    t = torch.add(t1,t2)
+    t = torch.add(t1, t2)
     t = t.sum(dim=1)
     loss = t / n_values.shape[1]
     loss = loss.sum(dim=0)
         self.embedding = Embedding(itemnum, parameter)
 
         self.relation_learner = MetaLearner(parameter['K'] - 1, embed_size=self.embed_dim, num_hidden1=500,
-                                                    num_hidden2=200, out_size=100, dropout_p=0)
+                                            num_hidden2=200, out_size=100, dropout_p=0)
 
         self.embedding_learner = EmbeddingLearner()
-        # self.loss_func = nn.MarginRankingLoss(self.margin)
+        self.loss_func = nn.MarginRankingLoss(self.margin)
         # self.loss_func = bpr_max_loss
-        self.loss_func = bpr_loss
+        # self.loss_func = bpr_loss
 
         self.rel_q_sharing = dict()
 
                                 negative[:, :, 1, :]], 1).unsqueeze(2)
         return pos_neg_e1, pos_neg_e2
 
+    def fast_forward(self, tasks, curr_rel=''):
+        with torch.no_grad():
+            sup = self.embedding(tasks)
+            K = sup.shape[1]
+            rel_q = self.rel_q_sharing[curr_rel]
+            sup_neg_e1, sup_neg_e2 = sup[:, :, 0, :], sup[:, :, 1, :]
+            a = sup_neg_e1.cpu().detach().numpy()
+            b = rel_q.squeeze(1).cpu().detach().numpy()
+            b = np.tile(b, (1, a.shape[-2], 1))
+            c = sup_neg_e2.cpu().detach().numpy()
+            # print(a.shape,b.shape,c.shape)
+            scores = -LA.norm(a + b - c, 2, -1)
+            return scores
+
     def forward(self, task, iseval=False, curr_rel=''):
         # transfer task string into embedding
         support, support_negative, query, negative = [self.embedding(t) for t in task]
 
-        K = support.shape[1]              # num of K
+        K = support.shape[1]  # num of K
         num_sn = support_negative.shape[1]  # num of support negative
-        num_q = query.shape[1]              # num of query
+        num_q = query.shape[1]  # num of query
-        num_n = negative.shape[1]           # num of query negative
+        num_n = negative.shape[1]  # num of query negative
 
-        rel = self.relation_learner(support)
+        rel = self.relation_learner(support, iseval)
         rel.retain_grad()
 
-        rel_s = rel.expand(-1, K+num_sn, -1, -1)
+        rel_s = rel.expand(-1, K + num_sn, -1, -1)
 
         if iseval and curr_rel != '' and curr_rel in self.rel_q_sharing.keys():
             rel_q = self.rel_q_sharing[curr_rel]
 
             p_score, n_score = self.embedding_learner(sup_neg_e1, sup_neg_e2, rel_s, K)
 
-            # y = torch.Tensor([1]).to(self.device)
+            y = torch.Tensor([1]).to(self.device)
             self.zero_grad()
 
             # sorted,indecies = torch.sort(n_score, descending=True,dim=1)
             # n_values = sorted[:,0:p_score.shape[1]]
 
-            # loss = self.loss_func(p_score, n_values, y)
+            loss = self.loss_func(p_score, n_score, y)
-            loss = self.loss_func(p_score,n_score,device=self.device)
+            # loss = self.loss_func(p_score,n_score,device=self.device)
             loss.backward(retain_graph=True)
 
             grad_meta = rel.grad
-            rel_q = rel - self.beta*grad_meta
+            rel_q = rel - self.beta * grad_meta
             self.rel_q_sharing[curr_rel] = rel_q
 
         rel_q = rel_q.expand(-1, num_q + num_n, -1, -1)
 
-        que_neg_e1, que_neg_e2 = self.split_concat(query, negative)  
+        que_neg_e1, que_neg_e2 = self.split_concat(query, negative)
         p_score, n_score = self.embedding_learner(que_neg_e1, que_neg_e2, rel_q, num_q)
 
         return p_score, n_score

sampler.py

 import sys
 import copy
-import torch
 import random
 import numpy as np
 from collections import defaultdict, Counter
 from multiprocessing import Process, Queue
 
 
-def random_neq(l, r, s, user_train,usernum):
+def random_neq(l, r, s):
-    # t = np.random.randint(l, r)
+    t = np.random.randint(l, r)
-    # while t in s:
+    while t in s:
-    #     t = np.random.randint(l, r)
+        t = np.random.randint(l, r)
-    # return t
+    return t
-    # user = np.random.choice(1, usernum + 1)
 
-    # user = random.randint(1,usernum+1)
-    # while len(user_train[user])<3:
-    #     user = random.randint(1, usernum + 1)
-    # candid_item = user_train[user][random.randint(0, len(user_train[user])-1)]
-    #
-    # while candid_item in s:
-    #     while len(user_train[user]) < 3:
-    #         user = random.randint(1, usernum + 1)
-    #     candid_item = user_train[user][random.randint(0, len(user_train[user])-1)]
-    # return candid_item
 
-    user = random.choice(list(user_train.keys()))
+def sample_function_mixed(user_train, usernum, itemnum, batch_size, maxlen, result_queue, SEED):
-    item = random.choice(user_train[user])
+    def sample():
 
-    while item in s:
+        if random.random() < 0.5:
-        user = random.choice(list(user_train.keys()))
+            user = np.random.randint(1, usernum + 1)
-        item = random.choice(user_train[user])
+            while len(user_train[user]) <= 1: user = np.random.randint(1, usernum + 1)
-    return item
-
-def random_negetive_batch(l, r, s, user_train,usernum, batch_users):
-    user = np.random.choice(batch_users)
-    candid_item = user_train[user][np.random.randint(0, len(user_train[user]))]
-
-    while candid_item in s:
-        user = np.random.choice(batch_users)
-        candid_item = user_train[user][np.random.randint(0, len(user_train[user]))]
-    return candid_item
-
-
-def sample_function_mixed(user_train, usernum, itemnum, batch_size, maxlen, result_queue, SEED,number_of_neg):
-
-    def sample(user,batch_users):
-        if random.random()<=0.5:
-            # user = np.random.randint(1, usernum + 1)
-            # while len(user_train[user]) <= 1: user = np.random.randint(1, usernum + 1)
 
             seq = np.zeros([maxlen], dtype=np.int32)
             pos = np.zeros([maxlen], dtype=np.int32)
-            neg = np.zeros([(maxlen-1)*number_of_neg + 1], dtype=np.int32)
+            neg = np.zeros([maxlen], dtype=np.int32)
 
             if len(user_train[user]) < maxlen:
                 nxt_idx = len(user_train[user]) - 1
             else:
-                nxt_idx = np.random.randint(maxlen,len(user_train[user]))
+                nxt_idx = np.random.randint(maxlen, len(user_train[user]))
 
             nxt = user_train[user][nxt_idx]
             idx = maxlen - 1
 
             ts = set(user_train[user])
-            for i in reversed(user_train[user][(nxt_idx - maxlen) : nxt_idx ]):
+            for i in reversed(user_train[user][min(0, nxt_idx - 1 - maxlen): nxt_idx - 1]):
                 seq[idx] = i
                 pos[idx] = nxt
-                # if nxt != 0: neg[idx] = random_neq(1, itemnum + 1, ts, user_train,usernum)
+                if nxt != 0: neg[idx] = random_neq(1, itemnum + 1, ts)
                 nxt = i
                 idx -= 1
                 if idx == -1: break
 
-            for i in range(len(neg)):
-                # neg[i] = random_neq(1, itemnum + 1, ts, user_train,usernum)
-                neg[i] = random_negetive_batch(1, itemnum + 1, ts, user_train, usernum, batch_users = batch_users)
-
             curr_rel = user
             support_triples, support_negative_triples, query_triples, negative_triples = [], [], [], []
-            for idx in range(maxlen-1):
+            for idx in range(maxlen - 1):
-                support_triples.append([seq[idx],curr_rel,pos[idx]])
+                support_triples.append([seq[idx], curr_rel, pos[idx]])
-                # support_negative_triples.append([seq[idx],curr_rel,neg[idx]])
+                support_negative_triples.append([seq[idx], curr_rel, neg[idx]])
-                # support_negative_triples.append([seq[-1], curr_rel, neg[idx]])
+            query_triples.append([seq[-1], curr_rel, pos[-1]])
-
+            negative_triples.append([seq[-1], curr_rel, neg[-1]])
-            # for idx in range(maxlen*30 - 1):
-            #     support_negative_triples.append([seq[-1], curr_rel, neg[idx]])
-            for j in range(number_of_neg):
-                for idx in range(maxlen-1):
-                    support_negative_triples.append([seq[idx], curr_rel, neg[j*(maxlen-1) + idx]])
-
-            query_triples.append([seq[-1],curr_rel,pos[-1]])
-            negative_triples.append([seq[-1],curr_rel,neg[-1]])
 
             return support_triples, support_negative_triples, query_triples, negative_triples, curr_rel
 
         else:
-            # print("bug happened in sample_function_mixed")
+            user = np.random.randint(1, usernum + 1)
-            # user = np.random.randint(1, usernum + 1)
+            while len(user_train[user]) <= 1: user = np.random.randint(1, usernum + 1)
-            # while len(user_train[user]) <= 1: user = np.random.randint(1, usernum + 1)
 
             seq = np.zeros([maxlen], dtype=np.int32)
             pos = np.zeros([maxlen], dtype=np.int32)
-            neg = np.zeros([maxlen*number_of_neg], dtype=np.int32)
+            neg = np.zeros([maxlen], dtype=np.int32)
 
             list_idx = random.sample([i for i in range(len(user_train[user]))], maxlen + 1)
             list_item = [user_train[user][i] for i in sorted(list_idx)]
             for i in reversed(list_item[:-1]):
                 seq[idx] = i
                 pos[idx] = nxt
-                # if nxt != 0: neg[idx] = random_neq(1, itemnum + 1, ts)
+                if nxt != 0: neg[idx] = random_neq(1, itemnum + 1, ts)
                 nxt = i
                 idx -= 1
                 if idx == -1: break
 
             curr_rel = user
             support_triples, support_negative_triples, query_triples, negative_triples = [], [], [], []
-
+            for idx in range(maxlen - 1):
-            for i in range(len(neg)):
+                support_triples.append([seq[idx], curr_rel, pos[idx]])
-                # neg[i] = random_neq(1, itemnum + 1, ts, user_train,usernum)
+                support_negative_triples.append([seq[idx], curr_rel, neg[idx]])
-                neg[i] = random_negetive_batch(1, itemnum + 1, ts, user_train, usernum, batch_users = batch_users)
+            query_triples.append([seq[-1], curr_rel, pos[-1]])
-
+            negative_triples.append([seq[-1], curr_rel, neg[-1]])
-            for j in range(number_of_neg):
-                for idx in range(maxlen-1):
-                    support_negative_triples.append([seq[idx], curr_rel, neg[j*maxlen + idx]])
-
-            for idx in range(maxlen-1):
-                support_triples.append([seq[idx],curr_rel,pos[idx]])
-                # support_negative_triples.append([seq[idx],curr_rel,neg[idx]])
-            query_triples.append([seq[-1],curr_rel,pos[-1]])
-            negative_triples.append([seq[-1],curr_rel,neg[-1]])
 
             return support_triples, support_negative_triples, query_triples, negative_triples, curr_rel
 
     np.random.seed(SEED)
-    
+
     while True:
         one_batch = []
-
-        users = []
-        for i in range(batch_size):
-            user = np.random.randint(1, usernum + 1)
-            while len(user_train[user]) <= 1: user = np.random.randint(1, usernum + 1)
-            users.append(user)
-
         for i in range(batch_size):
-            one_batch.append(sample(user = users[i], batch_users = users))
+            one_batch.append(sample())
 
         support, support_negative, query, negative, curr_rel = zip(*one_batch)
 
         result_queue.put(([support, support_negative, query, negative], curr_rel))
 
+
 class WarpSampler(object):
-    def __init__(self, User, usernum, itemnum, batch_size=64, maxlen=10, n_workers=1,params = None):
+    def __init__(self, User, usernum, itemnum, batch_size=64, maxlen=10, n_workers=1):
         self.result_queue = Queue(maxsize=n_workers * 10)
         self.processors = []
         for i in range(n_workers):
             self.processors.append(
                 Process(target=sample_function_mixed, args=(User,
-                                                      usernum,
+                                                            usernum,
-                                                      itemnum,
+                                                            itemnum,
-                                                      batch_size,
+                                                            batch_size,
-                                                      maxlen,
+                                                            maxlen,
-                                                      self.result_queue,
+                                                            self.result_queue,
-                                                      np.random.randint(2e9),
+                                                            np.random.randint(2e9)
-                                                      params['number_of_neg']
+                                                            )))
-                                                      )))
             self.processors[-1].daemon = True
             self.processors[-1].start()
 
         for p in self.processors:
             p.terminate()
             p.join()
-
-

trainer.py

 from models import *
 import os
 import sys
-import torch
 import shutil
 import logging
 import numpy as np
+import random
+import copy
+from operator import itemgetter
+import gc
 
 
 class Trainer:
-    def __init__(self, data_loaders, itemnum, parameter):
+    def __init__(self, data_loaders, itemnum, parameter, user_train_num, user_train):
+        # print(user_train)
         self.parameter = parameter
         # data loader
         self.train_data_loader = data_loaders[0]
         self.batch_size = parameter['batch_size']
         self.learning_rate = parameter['learning_rate']
         self.epoch = parameter['epoch']
-        # self.print_epoch = parameter['print_epoch']
-        # self.eval_epoch = parameter['eval_epoch']
-        # self.device = torch.device(parameter['device'])
         self.device = parameter['device']
-
         self.MetaTL = MetaTL(itemnum, parameter)
         self.MetaTL.to(parameter['device'])
-
         self.optimizer = torch.optim.Adam(self.MetaTL.parameters(), self.learning_rate)
-
         if parameter['eval_epoch']:
             self.eval_epoch = parameter['eval_epoch']
         else:
             self.eval_epoch = 1000
 
+        self.varset_size = parameter['varset_size']
+        self.user_train = user_train
+        self.warmup = parameter['warmup']
+        self.alpha = parameter['alpha']
+        self.S1 = parameter['S1']
+        self.S2_div_S1 = parameter['S2_div_S1']
+        self.temperature = parameter['temperature']
+        self.itemnum = itemnum
+        self.user_train_num = user_train_num
+        # init the two candidate sets for monitoring variance
+        self.candidate_cur = np.random.choice(itemnum, [user_train_num + 1, self.varset_size])
+        # for i in range(1,user_train_num+1):
+        #     for j in range(self.varset_size):
+        #         while self.candidate_cur[i, j] in user_train[i]:
+        #             self.candidate_cur[i, j] = random.randint(1, itemnum)
+
+        # self.candidate_nxt = [np.random.choice(itemnum, [user_train_num+1, self.varset_size]) for _ in range(5)]
+        # for c in range(5):
+        #     for i in range(1,user_train_num+1):
+        #         for j in range(self.varset_size):
+        #             while self.candidate_nxt[c][i, j] in user_train[i]:
+        #                 self.candidate_nxt[c][i, j] = random.randint(1, itemnum)
+
+        self.Mu_idx = {}
+        for i in range(user_train_num + 1):
+            Mu_idx_tmp = random.sample(list(range(self.varset_size)), self.S1)
+            self.Mu_idx[i] = Mu_idx_tmp
+
+        # todo : calculate score of positive items
+        self.score_cand_cur = {}
+        self.score_pos_cur = {}
+
+        # final candidate after execution of change_mu (after one_step) (for later epochs)
+        self.final_negative_items = {}
+
+    def change_mu(self, p_score, n_score, epoch_cur, users, train_task):
+        negitems = {}
+        negitems_candidates_all = {}
+
+        # for i in users:
+        #     negitems_candidates_all[i] = self.Mu_idx[i]
+        negitems_candidates_all = self.Mu_idx.copy()
+
+        ratings_positems = p_score.cpu().detach().numpy()
+        ratings_positems = np.reshape(ratings_positems, [-1])
+
+        # added
+        cnt = 0
+        for i in users:
+            self.score_pos_cur[i] = ratings_positems[cnt]
+            cnt += 1
+
+        Mu_items_all = {index: value[negitems_candidates_all[i]] for index, value in enumerate(self.candidate_cur)}
+
+        task = np.array(train_task[2])
+        task = np.tile(task, reps=(1, self.S1, 1))
+        task[:, :, 2] = np.array(itemgetter(*users)(Mu_items_all))
+
+        ratings_candidates_all = self.MetaTL.fast_forward(task, users)
+
+        hisscore_candidates_all = [self.score_cand_cur[i][:, negitems_candidates_all[i]] for user in users]
+        hisscore_pos_all = ratings_positems.copy()
+
+        hisscore_candidates_all = np.array(hisscore_candidates_all).transpose((1, 0, 2))
+
+        hisscore_pos_all = np.array(hisscore_pos_all)
+        hisscore_pos_all = hisscore_pos_all[:, np.newaxis]
+        hisscore_pos_all = np.tile(hisscore_pos_all, (hisscore_candidates_all.shape[0], 1, 1))
+
+        hislikelihood_candidates_all = 1 / (1 + np.exp(hisscore_pos_all - hisscore_candidates_all))
+
+        mean_candidates_all = np.mean(hislikelihood_candidates_all[:, :], axis=0)
+        variance_candidates_all = np.zeros(mean_candidates_all.shape)
+        for i in range(hislikelihood_candidates_all.shape[0]):
+            variance_candidates_all += (hislikelihood_candidates_all[i, :, :] - mean_candidates_all) ** 2
+        variance_candidates_all = np.sqrt(variance_candidates_all / hislikelihood_candidates_all.shape[0])
+
+        likelihood_candidates_all = \
+            1 / (1 + np.exp(np.expand_dims(ratings_positems, -1) - ratings_candidates_all))
+
+        # Top sampling strategy by score + alpha * std
+        item_arg_all = None
+        if self.alpha >= 0:
+            # item_arg_all = np.argmax(likelihood_candidates_all +
+            #                          self.alpha * min(1, epoch_cur / self.warmup)
+            #                          * variance_candidates_all, axis=1)
+            a = likelihood_candidates_all + self.alpha * min(1, epoch_cur / self.warmup) * variance_candidates_all
+            item_arg_all = np.argpartition(a, kth=(-2), axis=1)
+            item_arg_all = np.array(item_arg_all)[:, -2:]
+        else:
+            item_arg_all = np.argmax(variance_candidates_all, axis=1)
+
+        # negitems = { user : self.candidate_cur[user][negitems_candidates_all[user][item_arg_all[index]]] for index,user in enumerate(users)}
+        negitems0 = {user: self.candidate_cur[user][negitems_candidates_all[user][item_arg_all[index][0]]] for
+                     index, user in enumerate(users)}
+        negitems1 = {user: self.candidate_cur[user][negitems_candidates_all[user][item_arg_all[index][1]]] for
+                     index, user in enumerate(users)}
+
+        ###############################
+        for i in users:
+            self.final_negative_items[i] = [negitems0[i], negitems1[i]]
+        ###############################
+
+        # update Mu
+        negitems_mu_candidates = {}
+
+        for i in users:
+            Mu_set = set(self.Mu_idx[i])
+
+            while len(self.Mu_idx[i]) < self.S1 * (1 + self.S2_div_S1):
+                random_item = random.randint(0, self.candidate_cur.shape[1] - 1)
+                while random_item in Mu_set:
+                    random_item = random.randint(0, self.candidate_cur.shape[1] - 1)
+                self.Mu_idx[i].append(random_item)
+
+            negitems_mu_candidates[i] = self.Mu_idx[i]
+
+        negitems_mu = {}
+        negitems_mu = {user: self.candidate_cur[user][negitems_mu_candidates[user]] for user in users}
+
+        task = np.array(train_task[2])
+        task = np.tile(task, reps=(1, self.S1 * (1 + self.S2_div_S1), 1))
+        task[:, :, 2] = np.array(itemgetter(*users)(negitems_mu))
+
+        ratings_mu_candidates = self.MetaTL.fast_forward(task, users)
+        ratings_mu_candidates = ratings_mu_candidates / self.temperature
+
+        if np.any(np.isnan(ratings_mu_candidates)):
+            print("nan happend in ratings_mu_candidates")
+            ratings_mu_candidates = np.nan_to_num(ratings_mu_candidates)
+
+        ratings_mu_candidates = np.exp(ratings_mu_candidates) / np.reshape(
+            np.sum(np.exp(ratings_mu_candidates), axis=1), [-1, 1])
+
+        if np.any(np.isnan(ratings_mu_candidates)):
+            print("nan happend__2 in ratings_mu_candidates")
+            ratings_mu_candidates = self.MetaTL.fast_forward(task, users)
+            ratings_mu_candidates = ratings_mu_candidates / self.temperature
+            ratings_mu_candidates = ratings_mu_candidates + 100
+            ratings_mu_candidates = np.exp(ratings_mu_candidates) / np.reshape(
+                np.sum(np.exp(ratings_mu_candidates), axis=1), [-1, 1])
+
+        user_set = set()
+        cnt = 0
+        for i in users:
+            if i in user_set:
+                continue
+            else:
+                user_set.add(i)
+            cache_arg = np.random.choice(self.S1 * (1 + self.S2_div_S1), self.S1,
+                                         p=ratings_mu_candidates[cnt], replace=False)
+            self.Mu_idx[i] = np.array(self.Mu_idx[i])[cache_arg].tolist()
+            cnt += 1
+
+        second_cand = 0
+        del negitems, ratings_positems, Mu_items_all, task, ratings_candidates_all, hisscore_candidates_all, hisscore_pos_all
+        del hislikelihood_candidates_all, mean_candidates_all, variance_candidates_all, likelihood_candidates_all, second_cand
+        del negitems_mu, ratings_mu_candidates, user_set
+        gc.collect()
+
+    def change_candidate(self, epoch_count):
+        score_1epoch_nxt = []
+        for c in range(5):
+            # todo: implement proper funciton
+            pred = self.MetaTL(self.MetaTL.rel_q_sharing.keys(), self.candidate_nxt[c])
+            score_1epoch_nxt.append(np.array(pred))
+
+            # score_1epoch_nxt.append(np.array(/
+            #     [EvalUser.predict_fast(model, sess, num_user, num_item, parallel_users=100,
+            #                            predict_data=candidate_nxt[c])]))
+        # score_1epoch_pos = np.array(
+        #     [EvalUser.predict_pos(model, sess, num_user, max_posid, parallel_users=100, predict_data=train_pos)])
+        # todo: implement proper function
+        score_1epoch_pos = self.MetaTL(user_train, train_data)
+
+        # delete the score_cand_cur[0,:,:] at the earlist timestamp
+        if epoch_count >= 5 or epoch_count == 0:
+            self.score_pos_cur = np.delete(self.score_pos_cur, 0, 0)
+
+        for c in range(5):
+            self.score_cand_nxt[c] = np.concatenate([self.score_cand_nxt[c], score_1epoch_nxt[c]], axis=0)
+        self.score_pos_cur = np.concatenate([self.score_pos_cur, score_1epoch_pos], axis=0)
+
+        score_cand_cur = np.copy(self.score_cand_nxt[0])
+        candidate_cur = np.copy(self.candidate_nxt[0])
+        for c in range(4):
+            self.candidate_nxt[c] = np.copy(self.candidate_nxt[c + 1])
+            self.score_cand_nxt[c] = np.copy(self.score_cand_nxt[c + 1])
+        self.candidate_nxt[4] = np.random.choice(list(range(1, self.itemnum)), [self.user_train_num, self.varset_size])
+        for i in range(self.user_train_num):
+            for j in range(self.varset_size):
+                while self.candidate_nxt[4][i, j] in self.user_train[i]:
+                    self.candidate_nxt[4][i, j] = random.randint(0, self.itemnum - 1)
+        self.score_cand_nxt[4] = np.delete(self.score_cand_nxt[4], list(range(5)), 0)
 
     def rank_predict(self, data, x, ranks):
         # query_idx is the idx of positive score
             data['NDCG@1'] += 1 / np.log2(rank + 1)
         data['MRR'] += 1.0 / rank
 
-    def do_one_step(self, task, iseval=False, curr_rel=''):
+    def do_one_step(self, task, iseval=False, curr_rel='', epoch=None, train_task=None, epoch_count=None):
         loss, p_score, n_score = 0, 0, 0
+
         if not iseval:
+            task_new = copy.deepcopy(np.array(task[2]))
+
+            cnt = 0
+
+            for user in curr_rel:
+                if user in self.final_negative_items:
+                    for index, t in enumerate(task[1][cnt]):
+                        if index % 2 == 0:
+                            t[2] = self.final_negative_items[user][0]
+                        else:
+                            t[2] = self.final_negative_items[user][1]
+                cnt += 1
+
             self.optimizer.zero_grad()
             p_score, n_score = self.MetaTL(task, iseval, curr_rel)
             y = torch.Tensor([1]).to(self.device)
-            loss = self.MetaTL.loss_func(p_score, n_score,self.device)
+            loss = self.MetaTL.loss_func(p_score, n_score, y)
             loss.backward()
             self.optimizer.step()
+
+            # task_new = np.array(task[2])
+            task_new = np.tile(task_new, reps=(1, self.varset_size, 1))
+            task_new[:, :, 2] = np.array(itemgetter(*curr_rel)(self.candidate_cur))
+            data = self.MetaTL.fast_forward(task_new, curr_rel)
+
+            # prepare score_cand_cur (make all users to have the same number of history scores)
+            temp = min(epoch_count, 4)
+            for index, user in enumerate(curr_rel):
+                if (not user in self.score_cand_cur):
+                    self.score_cand_cur[user] = np.array([data[index]])
+                elif len(self.score_cand_cur[user]) <= temp:
+                    self.score_cand_cur[user] = np.concatenate(
+                        [self.score_cand_cur[user], np.array([data[index]])], axis=0)
+
+            self.change_mu(p_score, n_score, epoch_count, curr_rel, task)
+
         elif curr_rel != '':
             p_score, n_score = self.MetaTL(task, iseval, curr_rel)
             y = torch.Tensor([1]).to(self.device)
-            loss = self.MetaTL.loss_func(p_score, n_score,self.device)
+            loss = self.MetaTL.loss_func(p_score, n_score, y)
         return loss, p_score, n_score
 
     def train(self):
         best_epoch = 0
         best_value = 0
         bad_counts = 0
+        epoch_count = 0
 
         # training by epoch
         for e in range(self.epoch):
+            if e % 10 == 0: print("epoch:", e)
+
             # sample one batch from data_loader
             train_task, curr_rel = self.train_data_loader.next_batch()
-            loss, _, _ = self.do_one_step(train_task, iseval=False, curr_rel=curr_rel)
+            # change task negative samples using mu_idx
+            loss, _, _ = self.do_one_step(train_task, iseval=False, curr_rel=curr_rel, epoch=e, train_task=train_task,
+                                          epoch_count=epoch_count)
+
+            # after ten epoch epoch
+            if (e % 2500 == 0) and e != 0:
+                # init the two candidate sets for monitoring variance
+                self.candidate_cur = np.random.choice(self.itemnum, [self.user_train_num + 1, self.varset_size])
+                for i in range(1, self.user_train_num + 1):
+                    for j in range(self.varset_size):
+                        while self.candidate_cur[i, j] in self.user_train[i]:
+                            self.candidate_cur[i, j] = random.randint(1, self.itemnum)
+
+                self.Mu_idx = {}
+                for i in range(self.user_train_num + 1):
+                    Mu_idx_tmp = random.sample(list(range(self.varset_size)), self.S1)
+                    self.Mu_idx[i] = Mu_idx_tmp
+
+                self.score_cand_cur = {}
+                self.score_pos_cur = {}
+                self.final_negative_items = {}
+
+                # reset epoch_count has many effects on the chnage_mu and one_step and train function
+                epoch_count = 0
+
+            # after one epoch
+            elif e % 25 == 0 and e != 0:
+                self.check_complenteness(epoch_count)
+                print("epoch_count:", epoch_count)
+                print("=========================\n\n")
+                epoch_count += 1
 
             # do evaluation on specific epoch
             if e % self.eval_epoch == 0 and e != 0:
 
                 print('Epoch  {} Testing...'.format(e))
                 test_data = self.eval(istest=True, epoch=e)
-                
+                # original = r'/content/results.txt'
+                # target = r'/content/drive/MyDrive/MetaTL/MetaTL_v3/results.txt'
+                # shutil.copyfile(original, target)
+
+            # print(self.candidate_cur[curr_rel[0]],self.score_cand_cur[curr_rel[0]])
         print('Finish')
 
     def eval(self, istest=False, epoch=None):
-        # self.MetaTL.eval()
+        torch.backends.cudnn.enabled = False
-        
+        self.MetaTL.eval()
+
         self.MetaTL.rel_q_sharing = dict()
 
         if istest:
             # print current temp data dynamically
             for k in data.keys():
                 temp[k] = data[k] / t
-            # sys.stdout.write("{}\tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
-            #     t, temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))
-            # sys.stdout.flush()
 
         # print overall evaluation result and return it
         for k in data.keys():
             data[k] = round(data[k] / t, 3)
 
-        
+        print("\n")
         if istest:
-            print("TEST: \t test_loss: ",total_loss.detach().item())
+            print("TEST: \t test_loss: ", total_loss.detach().item())
-            print("TEST: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
+            print(
-                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']),"\n")
+                "TEST: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
+                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'],
+                    temp['Hits@1']))
             with open('results.txt', 'a') as f:
-                f.writelines("TEST: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r\n\n".format(
+                f.writelines(
-                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))
+                    "TEST: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r\n\n".format(
+                        temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'],
+                        temp['Hits@1']))
         else:
-            print("VALID: \t validation_loss: ", total_loss.detach().item() )
+            print("VALID: \t validation_loss: ", total_loss.detach().item())
-            print("VALID: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
+            print(
-                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))
+                "VALID: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
-            with open("results.txt",'a') as f:
+                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'],
-                f.writelines("VALID: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
+                    temp['Hits@1']))
-                    temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'], temp['Hits@1']))
+            with open("results.txt", 'a') as f:
-
+                f.writelines(
-        return data
+                    "VALID: \tMRR: {:.3f}\tNDCG@10: {:.3f}\tNDCG@5: {:.3f}\tNDCG@1: {:.3f}\tHits@10: {:.3f}\tHits@5: {:.3f}\tHits@1: {:.3f}\r".format(
+                        temp['MRR'], temp['NDCG@10'], temp['NDCG@5'], temp['NDCG@1'], temp['Hits@10'], temp['Hits@5'],
+                        temp['Hits@1']))
+
+        print("\n")
+        del total_loss, p_score, n_score
+        gc.collect()
+        self.MetaTL.train()
+        torch.backends.cudnn.enabled = True
+        return data
+
+    def check_complenteness(self, epoch_count):
+
+        # un_users = set()
+
+        for user in list(self.user_train.keys()):
+            if not user in self.score_cand_cur:
+                self.score_cand_cur[user] = np.array([np.zeros(self.varset_size)])
+
+            num = epoch_count - len(self.score_cand_cur[user]) + 1
+            if num > 0 and len(self.score_cand_cur[user]) < 5:
+                # if num!=1 : print("bug happend1")
+                # un_users.add(user)
+                self.score_cand_cur[user] = np.concatenate(
+                    [self.score_cand_cur[user], np.array([self.score_cand_cur[user][-1]])], axis=0)
+
+        if epoch_count >= 4:
+            t = 0
+            for user in list(self.score_cand_cur.keys()):
+                t = user
+                # self.score_cand_cur[user] = np.delete(self.score_cand_cur[user], 0, 0)
+                self.score_cand_cur[user] = self.score_cand_cur[user][-4:]

utils.py

 from multiprocessing import Process, Queue
 
 # sampler for batch generation
-def random_neq(l, r, s,user_train):
+def random_neq(l, r, s):
-    # t = np.random.randint(l, r)
+    t = np.random.randint(l, r)
-    # while t in s:
+    while t in s:
-    #     t = np.random.randint(l, r)
+        t = np.random.randint(l, r)
-    # return t
+    return t
 
-    user = random.choice(list(user_train.keys()))
+    # user = random.choice(list(user_train.keys()))
-    item = random.choice(user_train[user])
+    # item = random.choice(user_train[user])
-
+    #
-    while item in s:
+    # while item in s:
-        user = random.choice(list(user_train.keys()))
+    #     user = random.choice(list(user_train.keys()))
-        item = random.choice(user_train[user])
+    #     item = random.choice(user_train[user])
-    return item
+    # return item
 
 
 def trans_to_cuda(variable):
         
         self.itemnum = itemnum
 
-        if parameter['number_of_neg']:
+        # if parameter['number_of_neg']:
-            self.number_of_neg = parameter['number_of_neg']
+        #     self.number_of_neg = parameter['number_of_neg']
-        else:
+        # else:
-            self.number_of_neg = 5
+        #     self.number_of_neg = 5
 
 
     def next_one_on_eval(self):
         
         seq = np.zeros([self.maxlen], dtype=np.int32)
         pos = np.zeros([self.maxlen - 1], dtype=np.int32)
-        neg = np.zeros([self.maxlen * self.number_of_neg], dtype=np.int32)
+        # neg = np.zeros([self.maxlen * self.number_of_neg], dtype=np.int32)
-        
+        neg = np.zeros([self.maxlen - 1], dtype=np.int32)
+
         idx = self.maxlen - 1
 
         ts = set(self.train[u])
             seq[idx] = i
             if idx > 0:
                 pos[idx - 1] = i
-                # if i != 0: neg[idx - 1] = random_neq(1, self.itemnum + 1, ts,self.train)
+                if i != 0: neg[idx - 1] = random_neq(1, self.itemnum + 1, ts)
             idx -= 1
             if idx == -1: break
 
-        for i in range(len(neg)):
+        # for i in range(len(neg)):
-            neg[i] = random_neq(1, self.itemnum + 1, ts,self.train)
+        #     neg[i] = random_neq(1, self.itemnum + 1, ts,self.train)
 
         curr_rel = u
         support_triples, support_negative_triples, query_triples, negative_triples = [], [], [], []
         for idx in range(self.maxlen-1):
             support_triples.append([seq[idx],curr_rel,pos[idx]])
-            # support_negative_triples.append([seq[idx],curr_rel,neg[idx]])
+            support_negative_triples.append([seq[idx],curr_rel,neg[idx]])
             # support_negative_triples.append([seq[-1],curr_rel,neg[idx]])
 
         # for idx in range(len(neg)):
         #     support_negative_triples.append([seq[-1],curr_rel,neg[idx]])
         # print("injam",self.maxlen,list(range(self.maxlen-1)))
         # print("====")
-        for j in range(self.number_of_neg):
+        # for j in range(self.number_of_neg):
-            for idx in range(self.maxlen-1):
+        #     for idx in range(self.maxlen-1):
-                # print(j * self.maxlen + idx)
+        #         # print(j * self.maxlen + idx)
-                support_negative_triples.append([seq[idx], curr_rel, neg[j * (self.maxlen-1) + idx]])
+        #         support_negative_triples.append([seq[idx], curr_rel, neg[j * (self.maxlen-1) + idx]])
         # print("=end=\n\n")
 
         rated = ts