import math import time import torch import numpy as np import scipy as sp import pandas as pd import torch.nn as nn import networkx as nx import seaborn as sns from torch import Tensor import torch.nn.init as init import torch.nn.functional as F from torch.nn.parameter import Parameter from torch.nn.modules.module import Module if torch.cuda.is_available(): torch.device('cuda') """ Utils: Data Loader Feature Matrix Constructor Random Node Remover """ def Graph_load_batch(min_num_nodes=20, max_num_nodes=1000, name='ENZYMES', node_attributes=True, graph_labels=True): ''' load many graphs, e.g. enzymes :return: a list of graphs ''' print('Loading graph dataset: ' + str(name)) G = nx.Graph() # load data # path = '../dataset/' + name + '/' path = '/content/gdrive/My Drive/' + name + '/' data_adj = np.loadtxt(path + name + '_A.txt', delimiter=',').astype(int) if node_attributes: data_node_att = np.loadtxt(path + name + '_node_attributes.txt', delimiter=',') data_node_label = np.loadtxt(path + name + '_node_labels.txt', delimiter=',').astype(int) data_graph_indicator = np.loadtxt(path + name + '_graph_indicator.txt', delimiter=',').astype(int) if graph_labels: data_graph_labels = np.loadtxt(path + name + '_graph_labels.txt', delimiter=',').astype(int) data_tuple = list(map(tuple, data_adj)) G.add_edges_from(data_tuple) for i in range(data_node_label.shape[0]): if node_attributes: G.add_node(i + 1, feature=data_node_att[i]) G.add_node(i + 1, label=data_node_label[i]) G.remove_nodes_from(list(nx.isolates(G))) graph_num = data_graph_indicator.max() node_list = np.arange(data_graph_indicator.shape[0]) + 1 graphs = [] max_nodes = 0 for i in range(graph_num): nodes = node_list[data_graph_indicator == i + 1] G_sub = G.subgraph(nodes) if graph_labels: G_sub.graph['label'] = data_graph_labels[i] if G_sub.number_of_nodes() >= min_num_nodes and G_sub.number_of_nodes() <= max_num_nodes: graphs.append(G_sub) if G_sub.number_of_nodes() > max_nodes: max_nodes = G_sub.number_of_nodes() print('Loaded') return graphs def feature_matrix(g): ''' constructs the feautre matrix (N x 3) for the enzymes datasets ''' esm = nx.get_node_attributes(g, 'label') piazche = np.zeros((len(esm), 3)) for i, (k, v) in enumerate(esm.items()): piazche[i][v-1] = 1 return piazche # def remove_random_node(graph, max_size=40, min_size=10): # ''' # removes a random node from the gragh # returns the remaining graph matrix and the removed node links # ''' # if len(graph.nodes()) >= max_size or len(graph.nodes()) < min_size: # return None # relabeled_graph = nx.relabel.convert_node_labels_to_integers(graph) # choice = np.random.choice(list(relabeled_graph.nodes())) # remaining_graph = nx.to_numpy_matrix(relabeled_graph.subgraph(filter(lambda x: x != choice, list(relabeled_graph.nodes())))) # removed_node = nx.to_numpy_matrix(relabeled_graph)[choice] # graph_length = len(remaining_graph) # # source_graph = np.pad(remaining_graph, [(0, max_size - graph_length), (0, max_size - graph_length)]) # # target_graph = np.copy(source_graph) # removed_node_row = np.asarray(removed_node)[0] # # target_graph[graph_length] = np.pad(removed_node_row, [(0, max_size - len(removed_node_row))]) # return remaining_graph, removed_node_row def prepare_graph_data(graph, max_size=40, min_size=10): ''' gets a graph as an input returns a graph with a randomly removed node adj matrix [0], its feature matrix [1], the removed node true links [2] ''' if len(graph.nodes()) >= max_size or len(graph.nodes()) < min_size: return None relabeled_graph = nx.relabel.convert_node_labels_to_integers(graph) choice = np.random.choice(list(relabeled_graph.nodes())) remaining_graph = relabeled_graph.subgraph(filter(lambda x: x != choice, list(relabeled_graph.nodes()))) remaining_graph_adj = nx.to_numpy_matrix(remaining_graph) graph_length = len(remaining_graph) remaining_graph_adj = np.pad(remaining_graph_adj, [(0, max_size - graph_length), (0, max_size - graph_length)]) removed_node = nx.to_numpy_matrix(relabeled_graph)[choice] removed_node_row = np.asarray(removed_node)[0] removed_node_row = np.pad(removed_node_row, [(0, max_size - len(removed_node_row))]) return remaining_graph_adj, feature_matrix(remaining_graph), removed_node_row """" Layers: Graph Convolution Graph Multihead Attention Feed-Forward (as a MLP) """ class GraphConv(nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.input_dim = input_dim self.output_dim = output_dim self.weight = nn.Parameter(torch.FloatTensor(input_dim, output_dim).cuda()) self.relu = nn.ReLU() def forward(self, x, adj): ''' x is the feature matrix constructed in feature_matrix function adj ham ke is adjacency matrix of the graph ''' y = torch.matmul(adj, x) # print(y.shape) # print(self.weight.shape) y = torch.matmul(y, self.weight.double()) return y class GraphAttn(nn.Module): def __init__(self, heads, model_dim, dropout=0.1): super().__init__() self.model_dim = model_dim self.key_dim = model_dim // heads self.heads = heads self.q_linear = nn.Linear(model_dim, model_dim).cuda() self.v_linear = nn.Linear(model_dim, model_dim).cuda() self.k_linear = nn.Linear(model_dim, model_dim).cuda() self.dropout = nn.Dropout(dropout) self.out = nn.Linear(model_dim, model_dim).cuda() def forward(self, query, key, value): # print(q, k, v) bs = query.size(0) key = self.k_linear(key.float()).view(bs, -1, self.heads, self.key_dim) query = self.q_linear(query.float()).view(bs, -1, self.heads, self.key_dim) value = self.v_linear(value.float()).view(bs, -1, self.heads, self.key_dim) key = key.transpose(1,2) query = query.transpose(1,2) value = value.transpose(1,2) scores = attention(query, key, value, self.key_dim) concat = scores.transpose(1,2).contiguous().view(bs, -1, self.model_dim) output = self.out(concat) output = output.view(bs, self.model_dim) return output class FeedForward(nn.Module): def __init__(self, input_size, hidden_size): super().__init__() self.input_size = input_size self.hidden_size = hidden_size self.fully_connected1 = nn.Linear(self.input_size, self.hidden_size).cuda() self.relu = nn.ReLU() self.fully_connected2 = nn.Linear(self.hidden_size, 1).cuda() self.sigmoid = nn.Sigmoid() def forward(self, x): hidden = self.fully_connected1(x.float()) relu = self.relu(hidden) output = self.fully_connected2(relu) output = self.sigmoid(output) return output class Hydra(nn.Module): def __init__(self, gcn_input, model_dim, head): super().__init__() self.GCN = GraphConv(input_dim=gcn_input, output_dim=model_dim).cuda() self.GAT = GraphAttn(heads=head, model_dim=model_dim).cuda() self.MLP = FeedForward(input_size=model_dim, hidden_size=gcn_input).cuda() def forward(self, x, adj): gcn_outputs = self.GCN(x, adj) gat_output = self.GAT(gcn_outputs) mlp_output = self.MLP(gat_output).reshape(1,-1) return mlp_output """" Train the Model Prepare data using DataLoader (data can't be batched) """ def build_model(gcn_input, model_dim, head): model = Hydra(gcn_input, model_dim, head).cuda() return model def fn(batch): return batch[0] def train_model(model, trainloader, epoch, print_every=100): optim = torch.optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.98), eps=1e-9) model.train() start = time.time() temp = start total_loss = 0 for i in range(epoch): for batch, data in enumerate(trainloader, 0): adj, features, true_links = data adj, features, true_links = torch.tensor(adj).cuda(), torch.tensor(features).cuda(), torch.tensor(true_links).cuda() # print(adj.shape) # print(features.shape) # print(true_links.shape) preds = model(features, adj) optim.zero_grad() loss = F.binary_cross_entropy(preds.double(), true_links.double()) loss.backward() optim.step() total_loss += loss.item() if (i + 1) % print_every == 0: loss_avg = total_loss / print_every print("time = %dm, epoch %d, iter = %d, loss = %.3f,\ %ds per %d iters" % ((time.time() - start) // 60,\ epoch + 1, i + 1, loss_avg, time.time() - temp,\ print_every)) total_loss = 0 temp = time.time() # prepare data # coop = sum([list(filter(lambda x: x is not None, [prepare_graph_data(g) for g in graphs])) for i in range(10)], []) coop = list(filter(lambda x: x is not None, [prepare_graph_data(g) for g in graphs])) trainloader = torch.utils.data.DataLoader(coop, collate_fn=fn, batch_size=1) model = build_model(3, 243, 9) train_model(model, trainloader, 100, 10)