Graph_Transformer/GraphTransformer.py

import math
import torch
import torch.nn as nn
import torch.nn.init as init
import numpy as np
import pandas as pd
import networkx as nx
import scipy as sp
import seaborn as sns
# from node2vec import Node2Vec
from sklearn.decomposition import PCA
import copy
import time

if torch.cuda.is_available():
  torch.device('cuda')

"""Utils:
Data Loader / Attention / Clones / Embedder"""

def Graph_load_batch(min_num_nodes=20, max_num_nodes=1000, name='ENZYMES', node_attributes=True, graph_labels=True):
    '''
    load many graphs, e.g. enzymes
    :return: a list of graphs
    '''
    print('Loading graph dataset: ' + str(name))
    G = nx.Graph()
    # load data
    # path = '../dataset/' + name + '/'
    path = '/content/gdrive/My Drive/' + name + '/'
    data_adj = np.loadtxt(path + name + '_A.txt', delimiter=',').astype(int)
    if node_attributes:
        data_node_att = np.loadtxt(path + name + '_node_attributes.txt', delimiter=',')
    data_node_label = np.loadtxt(path + name + '_node_labels.txt', delimiter=',').astype(int)
    data_graph_indicator = np.loadtxt(path + name + '_graph_indicator.txt', delimiter=',').astype(int)
    if graph_labels:
        data_graph_labels = np.loadtxt(path + name + '_graph_labels.txt', delimiter=',').astype(int)
    data_tuple = list(map(tuple, data_adj))
    G.add_edges_from(data_tuple)
    for i in range(data_node_label.shape[0]):
        if node_attributes:
            G.add_node(i + 1, feature=data_node_att[i])
        G.add_node(i + 1, label=data_node_label[i])
    G.remove_nodes_from(list(nx.isolates(G)))
    graph_num = data_graph_indicator.max()
    node_list = np.arange(data_graph_indicator.shape[0]) + 1
    graphs = []
    max_nodes = 0
    for i in range(graph_num):
        nodes = node_list[data_graph_indicator == i + 1]
        G_sub = G.subgraph(nodes)
        if graph_labels:
            G_sub.graph['label'] = data_graph_labels[i]

        if G_sub.number_of_nodes() >= min_num_nodes and G_sub.number_of_nodes() <= max_num_nodes:
            graphs.append(G_sub)
            if G_sub.number_of_nodes() > max_nodes:
                max_nodes = G_sub.number_of_nodes()

    print('Loaded')
    return graphs

def attention(query, key, value, d_key):
  scores = torch.matmul(query, key.transpose(-2, -1)) /  math.sqrt(d_key)
  output = torch.matmul(scores, value)
  output = nn.functional.softmax(output)
  return output

def get_clones(module, N):
    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])

def embedder(graph, dimensions=32, walk_length=8, num_walks=200, workers=4):
  node2vec = Node2Vec(graph, dimensions=dimensions, walk_length=walk_length, num_walks=num_walks, workers=workers)  # Use temp_folder for big graphs
  model = node2vec.fit(window=10, min_count=1, batch_words=4)
  return model.wv.vectors

graphs = Graph_load_batch(min_num_nodes=10, name='ENZYMES')

# G = graphs[1]
# vecs = embedder(G)

# pca = PCA(n_components=2)
# principalComponents = pca.fit_transform(vecs)
# principalDf = pd.DataFrame(data = principalComponents
#              , columns = ['principal component 1', 'principal component 2'])
# principalDf.index = list(G.nodes())

# sns.scatterplot(principalDf['principal component 1'], principalDf['principal component 2'])

"""Sublayers"""

class MultiHeadAttention(nn.Module):
  def __init__(self, heads, d_model, dropout = 0.1):
    super().__init__()
    self.d_model = d_model
    self.d_k = d_model // heads
    self.h = heads
    self.q_linear = nn.Linear(d_model, d_model).cuda()
    self.v_linear = nn.Linear(d_model, d_model).cuda()
    self.k_linear = nn.Linear(d_model, d_model).cuda()
    self.dropout = nn.Dropout(dropout)
    self.out = nn.Linear(d_model, d_model)

  def forward(self, q, k, v):
    # print(q, k, v)
    bs = q.size(0)
    # perform linear operation and split into h heads
    k = self.k_linear(k).view(bs, -1, self.h, self.d_k)
    q = self.q_linear(q).view(bs, -1, self.h, self.d_k)
    v = self.v_linear(v).view(bs, -1, self.h, self.d_k)
    # transpose to get dimensions bs * h * sl * d_model
    k = k.transpose(1,2)
    q = q.transpose(1,2)
    v = v.transpose(1,2)

    scores = attention(q, k, v, self.d_k)
    # concatenate heads and put through final linear layer
    concat = scores.transpose(1,2).contiguous().view(bs, -1, self.d_model)
    output = self.out(concat)

    return output

class FeedForward(nn.Module):
  def __init__(self, d_model, d_ff=2048, dropout = 0.1):
    super().__init__()
    self.linear_1 = nn.Linear(d_model, d_ff).cuda()
    self.dropout = nn.Dropout(dropout)
    self.linear_2 = nn.Linear(d_ff, d_model).cuda()
    
  def forward(self, x):
    x = self.dropout(nn.functional.relu(self.linear_1(x)))
    x = self.linear_2(x)
    return x


class Norm(nn.Module):
  def __init__(self, d_model, eps = 1e-6):
    super().__init__()
    self.size = d_model
    self.alpha = nn.Parameter(torch.ones(self.size))
    self.bias = nn.Parameter(torch.zeros(self.size))
    self.eps = eps
    
  def forward(self, x):
    norm = self.alpha * (x - x.mean(dim=-1, keepdim=True)) / (x.std(dim=-1, keepdim=True) + self.eps) + self.bias
    return norm

"""Layers"""

class EncoderLayer(nn.Module):
  def __init__(self, d_model, heads, dropout = 0.1):
    super().__init__()
    self.norm_1 = Norm(d_model)
    self.norm_2 = Norm(d_model)
    self.attn = MultiHeadAttention(heads, d_model)
    self.ff = FeedForward(d_model)
    self.dropout_1 = nn.Dropout(dropout)
    self.dropout_2 = nn.Dropout(dropout)

  def forward(self, x):
    # x2 = self.norm_1(x)
    x = x + self.dropout_1(self.attn(x,x,x))
    # x2 = self.norm_2(x)
    x = x + self.dropout_2(self.ff(x))
    return x

class DecoderLayer(nn.Module):
  def __init__(self, d_model, heads, dropout=0.1):
    super().__init__()
    self.norm_1 = Norm(d_model)
    self.norm_2 = Norm(d_model)
    self.norm_3 = Norm(d_model)
    self.dropout_1 = nn.Dropout(dropout)
    self.dropout_2 = nn.Dropout(dropout)
    self.dropout_3 = nn.Dropout(dropout)
    self.attn_1 = MultiHeadAttention(heads, d_model)
    self.attn_2 = MultiHeadAttention(heads, d_model)
    self.ff = FeedForward(d_model).cuda()

  def forward(self, x, e_outputs):
    # x2 = self.norm_1(x)
    x = x + self.dropout_1(self.attn_1(x, x, x))
    # x2 = self.norm_2(x)
    # x2 = self.norm_2(x)
    x = x + self.dropout_2(self.attn_2(x, e_outputs, e_outputs))
    # x2 = self.norm_3(x)
    x = x + self.dropout_3(self.ff(x))
    return x

class Encoder(nn.Module):
  def __init__(self, vocab_size, d_model, N, heads):
    super().__init__()
    self.N = N
    self.layers = get_clones(EncoderLayer(d_model, heads), N)
    self.norm = Norm(d_model)
      
  def forward(self, src):
    x = src
    for i in range(N):
      x = self.layers[i](x)
    return self.norm(x)

class Decoder(nn.Module):
  def __init__(self, data_size, d_model, N, heads):
    super().__init__()
    self.N = N
    self.layers = get_clones(DecoderLayer(d_model, heads), N)
    self.norm = Norm(d_model)

  def forward(self, trg, e_outputs):
    x = trg
    for i in range(self.N):
      x = self.layers[i](x, e_outputs)
    return self.norm(x)

"""The Mighty Transformer"""

class Transformer(nn.Module):
  def __init__(self, src_graph, trg_graph, d_model, N, heads):
      super().__init__()
      self.encoder = Encoder(src_graph, d_model, N, heads)
      self.decoder = Decoder(trg_graph, d_model, N, heads)
      self.out = nn.Linear(d_model, trg_graph)

  def forward(self, src, trg):
      e_outputs = self.encoder(src)
      d_output = self.decoder(trg, e_outputs)
      output = self.out(d_output)
      return output


def remove_random_node(graph, max_size=40, min_size=10):
  if len(graph.nodes) >= max_size or len(graph.nodes) < min_size:
    return None
  relabeled_graph = nx.relabel.convert_node_labels_to_integers(graph)
  choice = np.random.choice(list(relabeled_graph.nodes))
  remaining_graph = nx.to_numpy_matrix(relabeled_graph.subgraph(filter(lambda x: x != choice, list(relabeled_graph.nodes))))
  removed_node = nx.to_numpy_matrix(relabeled_graph)[choice]
  graph_length = len(remaining_graph)
  source_graph = np.pad(remaining_graph, [(0, max_size - graph_length), (0, max_size - graph_length)])
  target_graph = np.copy(source_graph)
  removed_node_row = np.asarray(removed_node)[0]
  target_graph[graph_length] = np.pad(removed_node_row, [(0, max_size - len(removed_node_row))])
  return source_graph, target_graph

converted_graphs = list(filter(lambda x: x is not None, [remove_random_node(graph) for graph in graphs]))
source_graphs = torch.Tensor([graph[0] for graph in converted_graphs])
target_graphs = torch.Tensor([graph[1] for graph in converted_graphs])



d_model = 40
heads = 8
N = 6
src_size = len(source_graphs)
trg_size = len(target_graphs)

model = Transformer(src_size, trg_size, d_model, N, heads).cuda()

#print(model)

optim = torch.optim.Adam(model.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)


def train_model(epoch, print_every=100):
  model.train()
  start = time.time()
  temp = start
  total_loss = 0
  for i in range(epoch):

    src = source_graphs.cuda()
    trg = target_graphs.cuda()

    preds = model(src.float(), trg.float())
    optim.zero_grad()
    loss = torch.nn.functional.cross_entropy(preds.view(preds.size(-1), -1), trg.view(trg.size(0), -1))
    loss.backward()
    optim.step()
    total_loss += loss.data[0]
    if (i + 1) % print_every == 0:
      loss_avg = total_loss / print_every
      print("time = %dm, epoch %d, iter = %d, loss = %.3f,\
#       %ds per %d iters" % ((time.time() - start) // 60,\
      epoch + 1, i + 1, loss_avg, time.time() - temp,\
      print_every))
      total_loss = 0
      temp = time.time()

train_model(1, 1)

#preds = model(source_graphs[0].cuda(), target_graphs[0].cuda())
#loss = torch.nn.functional.cross_entropy(preds.view(preds.size(-1), -1), target_graphs.view(target_graphs.size(0), -1))
#