{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UIVN9SR710g1"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "hkaAUyQq2Dar",
    "outputId": "6eff7d9c-c145-480e-bcc6-239dbe287ecf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bKzFMiKU2FFC"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import time\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "import networkx as nx\n",
    "import seaborn as sns\n",
    "import torch.nn as nn\n",
    "from torch import Tensor\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.parameter import Parameter\n",
    "from torch.nn.modules.module import Module\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BLyNvPqk2REE"
   },
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "  torch.device('cuda')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "k_mb2ggqsZW5"
   },
   "source": [
    "# Data Loading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading graph dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Hjlmt3i42S20"
   },
   "outputs": [],
   "source": [
    "def Graph_load_batch(min_num_nodes=20, max_num_nodes=1000, name='ENZYMES', node_attributes=True, graph_labels=True):\n",
    "    print('Loading graph dataset: ' + str(name))\n",
    "    G = nx.Graph()\n",
    "    # load data\n",
    "    # path = '../dataset/' + name + '/'\n",
    "    path = '/content/gdrive/My Drive/' + name + '/'\n",
    "    data_adj = np.loadtxt(path + name + '_A.txt', delimiter=',').astype(int)\n",
    "    if node_attributes:\n",
    "        data_node_att = np.loadtxt(path + name + '_node_attributes.txt', delimiter=',')\n",
    "    data_node_label = np.loadtxt(path + name + '_node_labels.txt', delimiter=',').astype(int)\n",
    "    data_graph_indicator = np.loadtxt(path + name + '_graph_indicator.txt', delimiter=',').astype(int)\n",
    "    if graph_labels:\n",
    "        data_graph_labels = np.loadtxt(path + name + '_graph_labels.txt', delimiter=',').astype(int)\n",
    "    data_tuple = list(map(tuple, data_adj))\n",
    "    G.add_edges_from(data_tuple)\n",
    "    for i in range(data_node_label.shape[0]):\n",
    "        if node_attributes:\n",
    "            G.add_node(i + 1, feature=data_node_att[i])\n",
    "        G.add_node(i + 1, label=data_node_label[i])\n",
    "    G.remove_nodes_from(list(nx.isolates(G)))\n",
    "    graph_num = data_graph_indicator.max()\n",
    "    node_list = np.arange(data_graph_indicator.shape[0]) + 1\n",
    "    graphs = []\n",
    "    max_nodes = 0\n",
    "    for i in range(graph_num):\n",
    "        nodes = node_list[data_graph_indicator == i + 1]\n",
    "        G_sub = G.subgraph(nodes)\n",
    "        if graph_labels:\n",
    "            G_sub.graph['label'] = data_graph_labels[i]\n",
    "\n",
    "        if G_sub.number_of_nodes() >= min_num_nodes and G_sub.number_of_nodes() <= max_num_nodes:\n",
    "            graphs.append(G_sub)\n",
    "            if G_sub.number_of_nodes() > max_nodes:\n",
    "                max_nodes = G_sub.number_of_nodes()\n",
    "\n",
    "    print('Loaded')\n",
    "    return graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Constructing feature matrix of a graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7VMp_9e_2VNg"
   },
   "outputs": [],
   "source": [
    "def feature_matrix(g, max_nodes=40):\n",
    "    esm = nx.get_node_attributes(g, 'label')\n",
    "    piazche = np.zeros((max_nodes, 3))\n",
    "    for i, (k, v) in enumerate(esm.items()):\n",
    "        # print(i, k , v)\n",
    "        piazche[i][v-1] = 1\n",
    "    return piazche"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing a random node from a graph\n",
    "\n",
    "Returns remaining graph with removed node links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PkeAYLDQ2W5D"
   },
   "outputs": [],
   "source": [
    "def remove_random_node(graph, max_size=40, min_size=10):\n",
    "    if len(graph.nodes()) >= max_size or len(graph.nodes()) < min_size:\n",
    "        return None\n",
    "    relabeled_graph = nx.relabel.convert_node_labels_to_integers(graph)\n",
    "    choice = np.random.choice(list(relabeled_graph.nodes()))\n",
    "    remaining_graph = nx.to_numpy_matrix(relabeled_graph.subgraph(filter(lambda x: x != choice, list(relabeled_graph.nodes()))))\n",
    "    removed_node = nx.to_numpy_matrix(relabeled_graph)[choice]\n",
    "    graph_length = len(remaining_graph)\n",
    "    source_graph = np.pad(remaining_graph, [(0, max_size - graph_length), (0, max_size - graph_length)])\n",
    "    # target_graph = np.copy(source_graph)\n",
    "    removed_node_row = np.asarray(removed_node)[0]\n",
    "    # target_graph[graph_length] = np.pad(removed_node_row, [(0, max_size - len(removed_node_row))])\n",
    "    return remaining_graph, removed_node_row"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare graphs for the model\n",
    "\n",
    "returns a graph with a randomly removed node adj matrix [0], its feature matrix [1], the removed node true links [2] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FXFQJIvE2Ync"
   },
   "outputs": [],
   "source": [
    "def prepare_graph_data(graph, max_size=40, min_size=10):\n",
    "    if len(graph.nodes()) >= max_size or len(graph.nodes()) < min_size:\n",
    "        return None\n",
    "    relabeled_graph = nx.relabel.convert_node_labels_to_integers(graph)\n",
    "    choice = np.random.choice(list(relabeled_graph.nodes()))\n",
    "    remaining_graph = relabeled_graph.subgraph(filter(lambda x: x != choice, list(relabeled_graph.nodes())))\n",
    "    remaining_graph_adj = nx.to_numpy_matrix(remaining_graph)\n",
    "    graph_length = len(remaining_graph)\n",
    "    remaining_graph_adj = np.pad(remaining_graph_adj, [(0, max_size - graph_length), (0, max_size - graph_length)])\n",
    "    removed_node = nx.to_numpy_matrix(relabeled_graph)[choice]\n",
    "    removed_node_row = np.asarray(removed_node)[0]\n",
    "    removed_node_row = np.pad(removed_node_row, [(0, max_size - len(removed_node_row))])\n",
    "    return remaining_graph_adj, feature_matrix(remaining_graph),  removed_node_row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "jR7Zi1V52T-F",
    "outputId": "d3bbecaf-ee8a-41b6-8939-e2d0fb897ac8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading graph dataset: ENZYMES\n",
      "Loaded\n"
     ]
    }
   ],
   "source": [
    "graphs = Graph_load_batch(min_num_nodes=10, name='ENZYMES')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0pSj_CfaoSO6"
   },
   "outputs": [],
   "source": [
    "train, test = train_test_split(graphs, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tQdkjvY22_Kf"
   },
   "outputs": [],
   "source": [
    "# coop = sum([list(filter(lambda x: x is not None, [prepare_graph_data(g) for g in graphs])) for i in range(10)], [])\n",
    "coop = list(filter(lambda x: x is not None, [prepare_graph_data(g) for g in train]))\n",
    "dale = list(filter(lambda x: x is not None, [prepare_graph_data(g) for g in test]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qChhpZuCpHWv"
   },
   "outputs": [],
   "source": [
    "trainloader = torch.utils.data.DataLoader(coop, collate_fn=lambda x: x[0], batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8FHvrVFlqNh2"
   },
   "outputs": [],
   "source": [
    "testloader = torch.utils.data.DataLoader(dale, collate_fn=lambda x: x[0], batch_size=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JEvec2nosVn7"
   },
   "source": [
    "# Building Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Graph convolutional layer for extracting initial features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1qFtrFIJ2aoU"
   },
   "outputs": [],
   "source": [
    "class GraphConv(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super().__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.weight = nn.Parameter(torch.FloatTensor(input_dim, output_dim).cuda())\n",
    "        self.relu = nn.ReLU()\n",
    "\n",
    "    def forward(self, x, adj):\n",
    "        '''\n",
    "        x is hamun feature matrix \n",
    "        adj ham ke is hamun\n",
    "        '''\n",
    "        y = torch.matmul(adj, x)\n",
    "        # print(y.shape)\n",
    "        # print(self.weight.shape)\n",
    "        y = torch.matmul(y, self.weight.double())\n",
    "        return y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Attention calculator using given key, query and value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EnZo6dto2b6M"
   },
   "outputs": [],
   "source": [
    "def attention(query, key, value, key_dim):\n",
    "    # print('key:', key.transpose(-2,-1))\n",
    "    scores = torch.matmul(query, key.transpose(-2,-1)) / math.sqrt(key_dim)\n",
    "    scores = torch.matmul(scores, value)\n",
    "    # print('scores:', scores)\n",
    "    scores = F.softmax(scores)\n",
    "    # scores = torch.sigmoid(scores)\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Graph attention layer for more features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ORwhI8wm2eKO"
   },
   "outputs": [],
   "source": [
    "class GraphAttn(nn.Module):\n",
    "    def __init__(self, heads, model_dim, dropout=0.1):\n",
    "        super().__init__()\n",
    "        self.model_dim = model_dim\n",
    "        self.key_dim = model_dim // heads\n",
    "        self.heads = heads\n",
    "\n",
    "        self.q_linear = nn.Linear(model_dim, model_dim).cuda()\n",
    "        self.v_linear = nn.Linear(model_dim, model_dim).cuda()\n",
    "        self.k_linear = nn.Linear(model_dim, model_dim).cuda()\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.out = nn.Linear(model_dim, model_dim).cuda()\n",
    "\n",
    "    def forward(self, query, key, value):\n",
    "        # print(q, k, v)\n",
    "        bs = query.size(0)\n",
    "\n",
    "        key = self.k_linear(key.float()).view(bs, -1, self.heads, self.key_dim)\n",
    "        query = self.q_linear(query.float()).view(bs, -1, self.heads, self.key_dim)\n",
    "        value = self.v_linear(value.float()).view(bs, -1, self.heads, self.key_dim)\n",
    "\n",
    "        key = key.transpose(1,2)\n",
    "        query = query.transpose(1,2)\n",
    "        value = value.transpose(1,2)\n",
    "\n",
    "        scores = attention(query, key, value, self.key_dim)\n",
    "        concat = scores.transpose(1,2).contiguous().view(bs, -1, self.model_dim)\n",
    "        output = self.out(concat)\n",
    "        output = output.view(bs, self.model_dim)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MLP layer to map features to links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wKwSp9lx2fmW"
   },
   "outputs": [],
   "source": [
    "class FeedForward(nn.Module):\n",
    "        def __init__(self, input_size, hidden_size):\n",
    "            super().__init__()\n",
    "            self.input_size = input_size\n",
    "            self.hidden_size  = hidden_size\n",
    "            self.fully_connected1 = nn.Linear(self.input_size, self.hidden_size).cuda()\n",
    "            self.relu = nn.ReLU()\n",
    "            self.fully_connected2 = nn.Linear(self.hidden_size, 1).cuda()\n",
    "            self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "        def forward(self, x):\n",
    "            hidden = self.fully_connected1(x.float())\n",
    "            relu = self.relu(hidden)\n",
    "            output = self.fully_connected2(relu)\n",
    "            output = self.sigmoid(output)\n",
    "            return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assembeled model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hv6e81pz2hGU"
   },
   "outputs": [],
   "source": [
    "class Hydra(nn.Module):\n",
    "    def __init__(self, gcn_input, model_dim, head):\n",
    "        super().__init__()\n",
    "\n",
    "        self.GCN = GraphConv(input_dim=gcn_input, output_dim=model_dim).cuda()\n",
    "        self.GAT = GraphAttn(heads=head, model_dim=model_dim).cuda()\n",
    "        self.MLP = FeedForward(input_size=model_dim, hidden_size=gcn_input).cuda()\n",
    "\n",
    "    def forward(self, x, adj):\n",
    "        gcn_outputs = self.GCN(x, adj)\n",
    "        gat_output = self.GAT(gcn_outputs, gcn_outputs, gcn_outputs)\n",
    "        mlp_output = self.MLP(gat_output).reshape(1,-1)\n",
    "\n",
    "        return mlp_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Building model with given inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_4OIGkYs2iu8"
   },
   "outputs": [],
   "source": [
    "def build_model(gcn_input, model_dim, head):\n",
    "    model = Hydra(gcn_input, model_dim, head).cuda()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FxNdr2zR3Zgo"
   },
   "source": [
    "# Training Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training the model with given data and number of epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PEXlurWtpII2"
   },
   "outputs": [],
   "source": [
    "def train_model(model, trainloader, epoch, print_every=100):\n",
    "    optim = torch.optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.98), eps=1e-9)\n",
    "\n",
    "    model.train()\n",
    "    start = time.time()\n",
    "    temp = start\n",
    "    total_loss = 0\n",
    "\n",
    "    for i in range(epoch):\n",
    "        for batch, data in enumerate(trainloader, 0):\n",
    "            adj, features, true_links = data\n",
    "            adj, features, true_links  = torch.tensor(adj).cuda(), torch.tensor(features).cuda(), torch.tensor(true_links).cuda()\n",
    "            # print(adj.shape)\n",
    "            # print(features.shape)\n",
    "            # print(true_links.shape)\n",
    "            preds = model(features, adj)\n",
    "            optim.zero_grad()\n",
    "            loss = F.binary_cross_entropy(preds.double(), true_links.double())\n",
    "            writer.add_scalar('Loss/train', float(loss), i)\n",
    "            loss.backward()\n",
    "            optim.step()\n",
    "            total_loss += loss.item()\n",
    "        if (i + 1) % print_every == 0:\n",
    "            loss_avg = total_loss / print_every\n",
    "            print(\"time = %dm, epoch %d, iter = %d, loss = %.3f,\\\n",
    "            %ds per %d iters\" % ((time.time() - start) // 60,\\\n",
    "            epoch + 1, i + 1, loss_avg, time.time() - temp,\\\n",
    "            print_every))\n",
    "            total_loss = 0\n",
    "            temp = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0_N-yHrbo6C4"
   },
   "outputs": [],
   "source": [
    "kyle = build_model(3, 243, 9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 258
    },
    "colab_type": "code",
    "id": "Ax7DnN7ipK4A",
    "outputId": "84e06113-ba84-4995-97ae-8f26c43159bc"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:18: UserWarning: Using a target size (torch.Size([40])) that is different to the input size (torch.Size([1, 40])) is deprecated. Please ensure they have the same size.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time = 0m, epoch 101, iter = 10, loss = 92.819,        19s per 10 iters\n",
      "time = 0m, epoch 101, iter = 20, loss = 92.782,        19s per 10 iters\n",
      "time = 0m, epoch 101, iter = 30, loss = 92.814,        19s per 10 iters\n",
      "time = 1m, epoch 101, iter = 40, loss = 92.848,        19s per 10 iters\n",
      "time = 1m, epoch 101, iter = 50, loss = 92.723,        19s per 10 iters\n",
      "time = 1m, epoch 101, iter = 60, loss = 92.734,        19s per 10 iters\n",
      "time = 2m, epoch 101, iter = 70, loss = 92.718,        19s per 10 iters\n",
      "time = 2m, epoch 101, iter = 80, loss = 92.873,        19s per 10 iters\n",
      "time = 2m, epoch 101, iter = 90, loss = 92.730,        19s per 10 iters\n",
      "time = 3m, epoch 101, iter = 100, loss = 92.987,        19s per 10 iters\n"
     ]
    }
   ],
   "source": [
    "train_model(kyle, trainloader_train, 100, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing model and printing the loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qAPjB0KPpSlR"
   },
   "outputs": [],
   "source": [
    "def test_model(model, trainloader, print_every=10):\n",
    "    start = time.time()\n",
    "    temp = start\n",
    "    # total_loss = 0\n",
    "\n",
    "    for batch, data in enumerate(trainloader, 0):\n",
    "        adj, features, true_links = data\n",
    "        adj, features, true_links  = torch.tensor(adj).cuda(), torch.tensor(features).cuda(), torch.tensor(true_links).cuda()\n",
    "        # print(adj.shape)\n",
    "        # print(features.shape)\n",
    "        # print(true_links.shape)\n",
    "        preds = model(features, adj)\n",
    "        loss = F.binary_cross_entropy(preds.double(), true_links.double())\n",
    "        # total_loss += loss.item()\n",
    "        # loss_avg = total_loss / print_every\n",
    "        if (batch + 1) % print_every == 0:\n",
    "            print(\"loss = \", float(loss))\n",
    "            temp = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 224
    },
    "colab_type": "code",
    "id": "oKRQ0PtZra9Y",
    "outputId": "be66ebc4-909c-49cf-b1fb-acde9541cfb4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss =  0.3164060614241102\n",
      "loss =  0.4639609998843695\n",
      "loss =  0.2922176129342856\n",
      "loss =  0.31008866565259996\n",
      "loss =  0.21323858744621602\n",
      "loss =  0.22083598860733727\n",
      "loss =  0.3246467668552516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: UserWarning: Using a target size (torch.Size([40])) that is different to the input size (torch.Size([1, 40])) is deprecated. Please ensure they have the same size.\n",
      "  del sys.path[0]\n"
     ]
    }
   ],
   "source": [
    "test_model(kyle, trainloader_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "n-qj_vJXsK3n"
   },
   "source": [
    "# Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BgBoQGGVrt1p"
   },
   "outputs": [],
   "source": [
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FFYpLNF2s9Do"
   },
   "outputs": [],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AllQLpcPuln-"
   },
   "outputs": [],
   "source": [
    "writer = SummaryWriter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "vcotVmUgtdnR",
    "outputId": "4e3f9e47-c7e5-4917-a52b-859868dc1cbe"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "adj, features, true_links = next(iter(trainloader_train))\n",
    "adj, features, true_links  = torch.tensor(adj).cuda(), torch.tensor(features).cuda(), torch.tensor(true_links).cuda()\n",
    "grid = torchvision.utils.make_grid(adj)\n",
    "writer.add_image('adj', grid, 0)\n",
    "writer.add_graph(kyle, input_to_model= [features, adj])\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 821,
     "resources": {
      "https://localhost:6008/": {
       "data": "[encoded data, 740928 characters]",
       "headers": [
        [
         "content-type",
         "text/html; charset=utf-8"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/environment": {
       "data": "eyJkYXRhX2xvY2F0aW9uIjogInJ1bnMiLCAid2luZG93X3RpdGxlIjogIiJ9",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/experiments": {
       "data": "W10=",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugin/graphs/graph?run=Jun19_08-38-08_d89ac5b02a69&conceptual=false": {
       "data": "[encoded data, 44848 characters]",
       "headers": [
        [
         "content-type",
         "text/x-protobuf; charset=utf-8"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugin/graphs/info": {
       "data": "eyJKdW4xOV8wOC0zOC0wOF9kODlhYzViMDJhNjkiOiB7InJ1biI6ICJKdW4xOV8wOC0zOC0wOF9kODlhYzViMDJhNjkiLCAidGFncyI6IHsic3RlcDEiOiB7InRhZyI6ICJzdGVwMSIsICJjb25jZXB0dWFsX2dyYXBoIjogZmFsc2UsICJvcF9ncmFwaCI6IGZhbHNlLCAicHJvZmlsZSI6IHRydWV9fSwgInJ1bl9ncmFwaCI6IHRydWV9fQ==",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugin/images/images?run=Jun19_08-38-08_d89ac5b02a69&sample=0&tag=adj": {
       "data": "W3sid2FsbF90aW1lIjogMTU5MjU1NTg5NS43MDgwNTcsICJzdGVwIjogMCwgInF1ZXJ5IjogImJsb2Jfa2V5PVd5SWlMQ0pwYldGblpYTWlMQ0pLZFc0eE9WOHdPQzB6T0Mwd09GOWtPRGxoWXpWaU1ESmhOamtpTENKaFpHb2lMREFzTWwwIn1d",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugin/images/individualImage?ts=1592555895.708057&blob_key=WyIiLCJpbWFnZXMiLCJKdW4xOV8wOC0zOC0wOF9kODlhYzViMDJhNjkiLCJhZGoiLDAsMl0": {
       "data": "iVBORw0KGgoAAAANSUhEUgAAACgAAAAoCAIAAAADnC86AAAAhElEQVR4nO2VUQrAIAxD4+5/Z/cxEGmlyjSKkPc1HSSxZgwAcs4A/ENZBpv/qc18gn00T8mK0lVfPOQi2tUlTr6WNjn8K3qC7nLWKb5aimts1nRa/x2bTZNgpIOz9sGEj5WLZbmv0rEotVzPJ5RSGrxmFr7PRKfBBAf+lUIIIYQQQtzBC8bWyzqZ2BtSAAAAAElFTkSuQmCC",
       "headers": [
        [
         "content-type",
         "image/png"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugin/images/tags": {
       "data": "eyJKdW4xOV8wOC0zOC0wOF9kODlhYzViMDJhNjkiOiB7ImFkaiI6IHsiZGlzcGxheU5hbWUiOiAiYWRqIiwgImRlc2NyaXB0aW9uIjogIiIsICJzYW1wbGVzIjogMX19fQ==",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugin/scalars/scalars?tag=Loss%2Ftrain&run=Jun19_08-38-08_d89ac5b02a69": {
       "data": "[encoded data, 61444 characters]",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugin/scalars/tags": {
       "data": "eyJKdW4xOV8wOC0zOC0wOF9kODlhYzViMDJhNjkiOiB7Ikxvc3MvdHJhaW4iOiB7ImRpc3BsYXlOYW1lIjogIkxvc3MvdHJhaW4iLCAiZGVzY3JpcHRpb24iOiAiIn19fQ==",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/plugins_listing": {
       "data": "eyJzY2FsYXJzIjogeyJkaXNhYmxlX3JlbG9hZCI6IGZhbHNlLCAiZW5hYmxlZCI6IHRydWUsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJzY2FsYXJzIiwgImxvYWRpbmdfbWVjaGFuaXNtIjogeyJ0eXBlIjogIkNVU1RPTV9FTEVNRU5UIiwgImVsZW1lbnRfbmFtZSI6ICJ0Zi1zY2FsYXItZGFzaGJvYXJkIn19LCAiY3VzdG9tX3NjYWxhcnMiOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJDdXN0b20gU2NhbGFycyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtY3VzdG9tLXNjYWxhci1kYXNoYm9hcmQifX0sICJpbWFnZXMiOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogdHJ1ZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogImltYWdlcyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtaW1hZ2UtZGFzaGJvYXJkIn19LCAiYXVkaW8iOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJhdWRpbyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtYXVkaW8tZGFzaGJvYXJkIn19LCAiZGVidWdnZXIiOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJkZWJ1Z2dlciIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtZGVidWdnZXItZGFzaGJvYXJkIn19LCAiZ3JhcGhzIjogeyJkaXNhYmxlX3JlbG9hZCI6IHRydWUsICJlbmFibGVkIjogdHJ1ZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogImdyYXBocyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtZ3JhcGgtZGFzaGJvYXJkIn19LCAiZGlzdHJpYnV0aW9ucyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogImRpc3RyaWJ1dGlvbnMiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiQ1VTVE9NX0VMRU1FTlQiLCAiZWxlbWVudF9uYW1lIjogInRmLWRpc3RyaWJ1dGlvbi1kYXNoYm9hcmQifX0sICJoaXN0b2dyYW1zIjogeyJkaXNhYmxlX3JlbG9hZCI6IGZhbHNlLCAiZW5hYmxlZCI6IGZhbHNlLCAicmVtb3ZlX2RvbSI6IGZhbHNlLCAidGFiX25hbWUiOiAiaGlzdG9ncmFtcyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtaGlzdG9ncmFtLWRhc2hib2FyZCJ9fSwgInRleHQiOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogZmFsc2UsICJ0YWJfbmFtZSI6ICJ0ZXh0IiwgImxvYWRpbmdfbWVjaGFuaXNtIjogeyJ0eXBlIjogIkNVU1RPTV9FTEVNRU5UIiwgImVsZW1lbnRfbmFtZSI6ICJ0Zi10ZXh0LWRhc2hib2FyZCJ9fSwgInByX2N1cnZlcyI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogIlBSIEN1cnZlcyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtcHItY3VydmUtZGFzaGJvYXJkIn19LCAicHJvZmlsZV9yZWRpcmVjdCI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogIlByb2ZpbGUiLCAibG9hZGluZ19tZWNoYW5pc20iOiB7InR5cGUiOiAiQ1VTVE9NX0VMRU1FTlQiLCAiZWxlbWVudF9uYW1lIjogInRmLXByb2ZpbGUtcmVkaXJlY3QtZGFzaGJvYXJkIn19LCAiYmVob2xkZXIiOiB7ImRpc2FibGVfcmVsb2FkIjogZmFsc2UsICJlbmFibGVkIjogZmFsc2UsICJyZW1vdmVfZG9tIjogdHJ1ZSwgInRhYl9uYW1lIjogImJlaG9sZGVyIiwgImxvYWRpbmdfbWVjaGFuaXNtIjogeyJ0eXBlIjogIkNVU1RPTV9FTEVNRU5UIiwgImVsZW1lbnRfbmFtZSI6ICJ0Zi1iZWhvbGRlci1kYXNoYm9hcmQifX0sICJocGFyYW1zIjogeyJkaXNhYmxlX3JlbG9hZCI6IGZhbHNlLCAiZW5hYmxlZCI6IGZhbHNlLCAicmVtb3ZlX2RvbSI6IGZhbHNlLCAidGFiX25hbWUiOiAiaHBhcmFtcyIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAidGYtaHBhcmFtcy1kYXNoYm9hcmQifX0sICJtZXNoIjogeyJkaXNhYmxlX3JlbG9hZCI6IGZhbHNlLCAiZW5hYmxlZCI6IGZhbHNlLCAicmVtb3ZlX2RvbSI6IGZhbHNlLCAidGFiX25hbWUiOiAibWVzaCIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJDVVNUT01fRUxFTUVOVCIsICJlbGVtZW50X25hbWUiOiAibWVzaC1kYXNoYm9hcmQifX0sICJwcm9qZWN0b3IiOiB7ImRpc2FibGVfcmVsb2FkIjogdHJ1ZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogInByb2plY3RvciIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJJRlJBTUUiLCAibW9kdWxlX3BhdGgiOiAiL2RhdGEvcGx1Z2luL3Byb2plY3Rvci9pbmRleC5qcyJ9fSwgIndoYXRpZiI6IHsiZGlzYWJsZV9yZWxvYWQiOiBmYWxzZSwgImVuYWJsZWQiOiBmYWxzZSwgInJlbW92ZV9kb20iOiBmYWxzZSwgInRhYl9uYW1lIjogIldoYXQtSWYgVG9vbCIsICJsb2FkaW5nX21lY2hhbmlzbSI6IHsidHlwZSI6ICJJRlJBTUUiLCAibW9kdWxlX3BhdGgiOiAiL2RhdGEvcGx1Z2luL3doYXRpZi9pbmRleC5qcyJ9fX0=",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/data/runs": {
       "data": "WyJKdW4xOV8wOC0zOC0wOF9kODlhYzViMDJhNjkiXQ==",
       "headers": [
        [
         "content-type",
         "application/json"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/font-roboto/RxZJdnzeo3R5zSexge8UUZBw1xU1rKptJj_0jans920.woff2": {
       "data": "[encoded data, 14384 characters]",
       "headers": [
        [
         "content-type",
         "font/woff2"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/font-roboto/d-6IYplOFocCacKzxwXSOJBw1xU1rKptJj_0jans920.woff2": {
       "data": "[encoded data, 14352 characters]",
       "headers": [
        [
         "content-type",
         "font/woff2"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/font-roboto/oMMgfZMQthOryQo9n22dcuvvDin1pK8aKteLpeZ5c0A.woff2": {
       "data": "[encoded data, 14332 characters]",
       "headers": [
        [
         "content-type",
         "font/woff2"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      },
      "https://localhost:6008/index.js": {
       "data": "[encoded data, 4445616 characters]",
       "headers": [
        [
         "content-type",
         "application/javascript; charset=utf-8"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      }
     }
    },
    "colab_type": "code",
    "id": "J1hBwApJtIwG",
    "outputId": "fe51e37f-f3fe-40ac-d56b-9f93e4721608"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "        (async () => {\n",
       "            const url = await google.colab.kernel.proxyPort(6008, {\"cache\": true});\n",
       "            const iframe = document.createElement('iframe');\n",
       "            iframe.src = url;\n",
       "            iframe.setAttribute('width', '100%');\n",
       "            iframe.setAttribute('height', '800');\n",
       "            iframe.setAttribute('frameborder', 0);\n",
       "            document.body.appendChild(iframe);\n",
       "        })();\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%tensorboard --logdir=runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UMZjWCfJutjI"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": " ibrahim.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
