{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## test adapt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from adapt.instance_based import KMM\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "X_source = np.random.randn(1000, 1) * 2 - 1\n",
    "Y_source = (X_source[:, 0] > -1) & (X_source[:, 0] < 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_target = np.random.randn(1000, 1) * 2 + 1\n",
    "Y_target = (X_target[:, 0] > -1.) & (X_target[:, 0] < 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.209"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "src_only = LogisticRegression(penalty=None)\n",
    "src_only.fit(X_source, Y_source)\n",
    "\n",
    "src_only.score(X_target, Y_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "adapt_model = KMM(\n",
    "    estimator=LogisticRegression(penalty=\"none\"), \n",
    "    Xt=X_target,\n",
    "    kernel=\"rbf\",  # Gaussian kernel\n",
    "    gamma=1.,     # Bandwidth of the kernel\n",
    "    verbose=0,\n",
    "    random_state=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.574"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adapt_model.fit(X_source, Y_source)\n",
    "adapt_model.score(X_target, Y_target)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'datasets/office31_labeled/amazon/labels.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[17], line 27\u001b[0m\n\u001b[1;32m     23\u001b[0m                     \u001b[39mprint\u001b[39m(j, image)\n\u001b[1;32m     24\u001b[0m                     \u001b[39mbreak\u001b[39;00m\n\u001b[0;32m---> 27\u001b[0m create_dataset_labels()\n",
      "Cell \u001b[0;32mIn[17], line 19\u001b[0m, in \u001b[0;36mcreate_dataset_labels\u001b[0;34m(base_path, out_path)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[39mfor\u001b[39;00m domain \u001b[39min\u001b[39;00m DOMAINS:\n\u001b[1;32m     17\u001b[0m     os\u001b[39m.\u001b[39mmakedirs(os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mjoin(out_path, domain), exist_ok\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)  \n\u001b[0;32m---> 19\u001b[0m     \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39;49m(os\u001b[39m.\u001b[39;49mpath\u001b[39m.\u001b[39;49mjoin(out_path, domain, \u001b[39m\"\u001b[39;49m\u001b[39mlabels.txt\u001b[39;49m\u001b[39m\"\u001b[39;49m)) \u001b[39mas\u001b[39;00m f:\n\u001b[1;32m     20\u001b[0m         \u001b[39mfor\u001b[39;00m i, class_name \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(os\u001b[39m.\u001b[39mlistdir(os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mjoin(base_path, domain))):\n\u001b[1;32m     21\u001b[0m             \u001b[39mprint\u001b[39m(i, class_name)\n",
      "File \u001b[0;32m~/miniconda3/envs/tf/lib/python3.8/site-packages/IPython/core/interactiveshell.py:282\u001b[0m, in \u001b[0;36m_modified_open\u001b[0;34m(file, *args, **kwargs)\u001b[0m\n\u001b[1;32m    275\u001b[0m \u001b[39mif\u001b[39;00m file \u001b[39min\u001b[39;00m {\u001b[39m0\u001b[39m, \u001b[39m1\u001b[39m, \u001b[39m2\u001b[39m}:\n\u001b[1;32m    276\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m    277\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mIPython won\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt let you open fd=\u001b[39m\u001b[39m{\u001b[39;00mfile\u001b[39m}\u001b[39;00m\u001b[39m by default \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    278\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    279\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39myou can use builtins\u001b[39m\u001b[39m'\u001b[39m\u001b[39m open.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    280\u001b[0m     )\n\u001b[0;32m--> 282\u001b[0m \u001b[39mreturn\u001b[39;00m io_open(file, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'datasets/office31_labeled/amazon/labels.txt'"
     ]
    }
   ],
   "source": [
    "import os \n",
    "\n",
    "DOMAINS = [\"amazon\", \"dslr\", \"webcam\"]\n",
    "\n",
    "def create_dataset_labels(base_path=\"datasets/office31\", out_path=\"datasets/office31_labeled\"):\n",
    "    \"\"\"\n",
    "    source_domains: List of domain(s) used for training and evaluating\n",
    "    target_domain: List of domain(s) used for testing\n",
    "    base_path: Base directory of office31 dataset. The structure of this directory should be as follows: \n",
    "                {domain_name}/{class_name}/{image_name}.jpg\n",
    "                e.g. amazon/bike_helmet/frame_0003.jpg\n",
    "    out_path: The path to store splitted data\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    for domain in DOMAINS:\n",
    "        os.makedirs(os.path.join(out_path, domain), exist_ok=True)  \n",
    "\n",
    "        with open(os.path.join(out_path, domain, \"labels.txt\")) as f:\n",
    "            for i, class_name in enumerate(os.listdir(os.path.join(base_path, domain))):\n",
    "                print(i, class_name)\n",
    "                for j, image in enumerate(os.listdir(os.path.join(base_path, domain, class_name))):\n",
    "                    print(j, image)\n",
    "                    break\n",
    "\n",
    "\n",
    "create_dataset_labels()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2817 files belonging to 31 classes.\n",
      "Using 2254 files for training.\n",
      "Using 563 files for validation.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "train_ds, val_ds = tf.keras.utils.image_dataset_from_directory(\n",
    "  \"datasets/office31/amazon\",\n",
    "  validation_split=0.2,\n",
    "  subset=\"both\",\n",
    "  seed=123,\n",
    "  image_size=(150, 150),\n",
    "  batch_size=32)\n",
    "\n",
    "class_names = train_ds.class_names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 228284 characters]",
      "text/plain": [
       "<Figure size 600x600 with 9 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "plt.figure(figsize=(6, 6))\n",
    "for images, labels in train_ds.take(1):\n",
    "  for i in range(9):\n",
    "    ax = plt.subplot(3, 3, i + 1)\n",
    "    plt.imshow(images[i].numpy().astype(\"uint8\"))\n",
    "    plt.title(class_names[labels[i]])\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 mug\n",
      "0 frame_0026.jpg\n",
      "1 phone\n",
      "0 frame_0012.jpg\n",
      "2 monitor\n",
      "0 frame_0026.jpg\n",
      "3 mobile_phone\n",
      "0 frame_0026.jpg\n",
      "4 paper_notebook\n",
      "0 frame_0026.jpg\n",
      "5 keyboard\n",
      "0 frame_0026.jpg\n",
      "6 bottle\n",
      "0 frame_0012.jpg\n",
      "7 stapler\n",
      "0 frame_0018.jpg\n",
      "8 scissors\n",
      "0 frame_0018.jpg\n",
      "9 punchers\n",
      "0 frame_0026.jpg\n",
      "10 laptop_computer\n",
      "0 frame_0026.jpg\n",
      "11 bookcase\n",
      "0 frame_0012.jpg\n",
      "12 printer\n",
      "0 frame_0018.jpg\n",
      "13 desktop_computer\n",
      "0 frame_0018.jpg\n",
      "14 letter_tray\n",
      "0 frame_0018.jpg\n",
      "15 ring_binder\n",
      "0 frame_0026.jpg\n",
      "16 back_pack\n",
      "0 frame_0026.jpg\n",
      "17 file_cabinet\n",
      "0 frame_0018.jpg\n",
      "18 headphones\n",
      "0 frame_0026.jpg\n",
      "19 desk_chair\n",
      "0 frame_0026.jpg\n",
      "20 speaker\n",
      "0 frame_0026.jpg\n",
      "21 projector\n",
      "0 frame_0026.jpg\n",
      "22 pen\n",
      "0 frame_0026.jpg\n",
      "23 trash_can\n",
      "0 frame_0018.jpg\n",
      "24 bike_helmet\n",
      "0 frame_0026.jpg\n",
      "25 bike\n",
      "0 frame_0018.jpg\n",
      "26 mouse\n",
      "0 frame_0026.jpg\n",
      "27 calculator\n",
      "0 frame_0026.jpg\n",
      "28 desk_lamp\n",
      "0 frame_0018.jpg\n",
      "29 ruler\n",
      "0 frame_0003.jpg\n",
      "30 tape_dispenser\n",
      "0 frame_0018.jpg\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "\n",
    "DOMAINS = [\"amazon\", \"dslr\", \"webcam\"]\n",
    "\n",
    "def split_dataset(source_domains=[], target_domains=[], base_path=\"datasets/office31\", out_path=\"datasets/office31_splitted\"):\n",
    "    \"\"\"\n",
    "    source_domains: List of domain(s) used for training and evaluating\n",
    "    target_domain: List of domain(s) used for testing\n",
    "    base_path: Base directory of office31 dataset. The structure of this directory should be as follows: \n",
    "                {domain_name}/{class_name}/{image_name}.jpg\n",
    "                e.g. amazon/bike_helmet/frame_0003.jpg\n",
    "    out_path: The path to store splitted data\n",
    "    \"\"\"\n",
    "\n",
    "    os.makedirs(os.path.join(out_path, \"source\"), exist_ok=True)\n",
    "    os.makedirs(os.path.join(out_path, \"target\"), exist_ok=True)\n",
    "\n",
    "    for domain in target_domains:\n",
    "        for i, class_name in enumerate(os.listdir(os.path.join(base_path, domain))):\n",
    "            print(i, class_name)\n",
    "            for j, image in enumerate(os.listdir(os.path.join(base_path, domain, class_name))):\n",
    "                print(j, image)\n",
    "                break\n",
    "\n",
    "\n",
    "split_dataset([\"amazon\"], [\"webcam\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for domain in [\"amazon\", \"dslr\", \"webcam\"]:\n",
    "    os.rename(\"office_dataset/{}/images\".format(domain), \"office31/train/{}\".format(domain))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for domain in [\"webcam\", \"dslr\"]:\n",
    "    os.makedirs(\"office31/val/{}\".format(domain))\n",
    "    os.makedirs(\"office31/test/{}\".format(domain))\n",
    "\n",
    "# Move a subset of the images to the validation and test sets\n",
    "for domain in [\"webcam\", \"dslr\"]:\n",
    "    for i, file in enumerate(os.listdir(\"office31/train/{}\".format(domain))):\n",
    "        if i % 10 == 0:\n",
    "            os.rename(\"office31/train/{}/{}\".format(domain, file), \"office31/val/{}/{}\".format(domain, file))\n",
    "        elif i % 10 == 1:\n",
    "            os.rename(\"office31/train/{}/{}\".format(domain, file), \"office31/test/{}/{}\".format(domain, file))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test adapt on office"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-14 11:01:16.482176: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data Saba's way"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2817 files belonging to 31 classes.\n",
      "Using 2254 files for training.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-14 11:01:22.530751: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1\n",
      "2021-05-14 11:01:22.537354: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.92GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-14 11:01:22.538192: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 1 with properties: \n",
      "pciBusID: 0000:0a:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.93GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-14 11:01:22.538216: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-05-14 11:01:22.539810: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10\n",
      "2021-05-14 11:01:22.541316: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10\n",
      "2021-05-14 11:01:22.541638: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10\n",
      "2021-05-14 11:01:22.543283: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10\n",
      "2021-05-14 11:01:22.544128: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10\n",
      "2021-05-14 11:01:22.547357: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-05-14 11:01:22.550681: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0, 1\n",
      "2021-05-14 11:01:22.551052: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-05-14 11:01:22.557188: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 3298130000 Hz\n",
      "2021-05-14 11:01:22.557609: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x64e5730 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2021-05-14 11:01:22.557623: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2021-05-14 11:01:22.648630: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x6551b60 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2021-05-14 11:01:22.648661: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): GeForce GTX TITAN X, Compute Capability 5.2\n",
      "2021-05-14 11:01:22.648668: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (1): GeForce GTX TITAN X, Compute Capability 5.2\n",
      "2021-05-14 11:01:22.650144: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.92GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-14 11:01:22.650808: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 1 with properties: \n",
      "pciBusID: 0000:0a:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.93GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-14 11:01:22.650842: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-05-14 11:01:22.650867: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10\n",
      "2021-05-14 11:01:22.650881: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10\n",
      "2021-05-14 11:01:22.650894: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10\n",
      "2021-05-14 11:01:22.650908: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10\n",
      "2021-05-14 11:01:22.650921: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10\n",
      "2021-05-14 11:01:22.650934: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-05-14 11:01:22.653499: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0, 1\n",
      "2021-05-14 11:01:22.653538: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-05-14 11:01:24.149550: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2021-05-14 11:01:24.149578: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263]      0 1 \n",
      "2021-05-14 11:01:24.149584: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0:   N Y \n",
      "2021-05-14 11:01:24.149588: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 1:   Y N \n",
      "2021-05-14 11:01:24.151857: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11021 MB memory) -> physical GPU (device: 0, name: GeForce GTX TITAN X, pci bus id: 0000:01:00.0, compute capability: 5.2)\n",
      "2021-05-14 11:01:24.152814: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 4063 MB memory) -> physical GPU (device: 1, name: GeForce GTX TITAN X, pci bus id: 0000:0a:00.0, compute capability: 5.2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2817 files belonging to 31 classes.\n",
      "Using 563 files for validation.\n",
      "Found 795 files belonging to 31 classes.\n"
     ]
    }
   ],
   "source": [
    "from data_loader import load_data, get_input_and_labels_from_batch_ds\n",
    "\n",
    "train_ds, val_ds, target_ds = load_data(\"amazon\", \"webcam\", image_size=(224,224))\n",
    "Xs, ys = get_input_and_labels_from_batch_ds(train_ds)\n",
    "Xv, yv = get_input_and_labels_from_batch_ds(val_ds)\n",
    "Xt, yt = get_input_and_labels_from_batch_ds(target_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: (2254, 224, 224, 3) (2254,)\n",
      "val: (563, 224, 224, 3) (563,)\n",
      "target: (795, 224, 224, 3) (795,)\n"
     ]
    }
   ],
   "source": [
    "print(\"train:\", Xs.shape, ys.shape)\n",
    "print(\"val:\", Xv.shape, yv.shape)\n",
    "print(\"target:\", Xt.shape, yt.shape)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data (official doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 62168 characters]",
      "text/plain": [
       "<Figure size 400x200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 106036 characters]",
      "text/plain": [
       "<Figure size 400x200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "path = \"datasets/office31/\" # path to downloaded Office dataset\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(4, 2))\n",
    "ax1.imshow(plt.imread(path+\"amazon/back_pack/frame_0001.jpg\"))\n",
    "ax2.imshow(plt.imread(path+\"amazon/bike/frame_0001.jpg\"))\n",
    "ax1.set_title(\"label = back-pack\"); ax2.set_title(\"label = bike\");\n",
    "plt.suptitle(\"Domain = Amazon\"); plt.show()\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(4, 2))\n",
    "ax1.imshow(plt.imread(path+\"webcam/back_pack/frame_0001.jpg\"))\n",
    "ax2.imshow(plt.imread(path+\"webcam/bike/frame_0001.jpg\"))\n",
    "ax1.set_title(\"label = back-pack\"); ax2.set_title(\"label = bike\");\n",
    "plt.suptitle(\"Domain = Webcam\"); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_12245/3952787103.py:15: DeprecationWarning: ANTIALIAS is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.LANCZOS instead.\n",
      "  image = image.resize((224, 224), Image.ANTIALIAS)\n"
     ]
    }
   ],
   "source": [
    "def get_Xy(domain, path_to_folder=path):\n",
    "\n",
    "    path = path_to_folder + domain \n",
    "    X = []\n",
    "    y = []\n",
    "\n",
    "    for r, d, f in os.walk(path):\n",
    "        for direct in d:\n",
    "            if not \".ipynb_checkpoints\" in direct:\n",
    "                for r, d, f in os.walk(os.path.join(path , direct)):\n",
    "                    for file in f:\n",
    "                        path_to_image = os.path.join(r, file)\n",
    "                        if not \".ipynb_checkpoints\" in path_to_image:\n",
    "                            image = Image.open(path_to_image)\n",
    "                            image = image.resize((224, 224), Image.ANTIALIAS)\n",
    "                            image = np.array(image, dtype=int)\n",
    "                            X.append(image)\n",
    "                            y.append(direct)\n",
    "    return X, y\n",
    "\n",
    "Xs, ys = get_Xy(\"amazon\")\n",
    "Xt, yt = get_Xy(\"webcam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2817 2817\n",
      "(224, 224, 3)\n",
      "mug\n"
     ]
    }
   ],
   "source": [
    "print(len(Xs), len(ys))\n",
    "print(Xs[0].shape)\n",
    "print(ys[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]] mug\n"
     ]
    }
   ],
   "source": [
    "print(Xs[0], ys[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  ...\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]]\n",
      "\n",
      " [[131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  ...\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]]\n",
      "\n",
      " [[131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  ...\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  ...\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]]\n",
      "\n",
      " [[131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  ...\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]]\n",
      "\n",
      " [[131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  ...\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]\n",
      "  [131.32    138.22101 151.061  ]]] 0\n"
     ]
    }
   ],
   "source": [
    "print(Xs_[0], Ys_[0])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ssl\n",
    "ssl._create_default_https_context = ssl._create_unverified_context\n",
    "\n",
    "from tensorflow.keras.applications.resnet50 import preprocess_input, ResNet50\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras.layers import Input\n",
    "\n",
    "# if you want to download weights, remove weights param in ResNet40 and remove this line\n",
    "WEIGHTS_PATH = 'model-weights/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'\n",
    "\n",
    "resnet50 = ResNet50(include_top=False, input_shape=(224, 224, 3), pooling=\"avg\", weights=WEIGHTS_PATH, classes=31)\n",
    "\n",
    "first_layer = resnet50.get_layer('conv5_block2_out')\n",
    "inputs = Input(first_layer.output_shape[1:])\n",
    "\n",
    "for layer in resnet50.layers[resnet50.layers.index(first_layer)+1:]:\n",
    "    if layer.name == \"conv5_block3_1_conv\":\n",
    "        x = layer(inputs)\n",
    "    elif layer.name == \"conv5_block3_add\":\n",
    "        x = layer([inputs, x])\n",
    "    else:\n",
    "        x = layer(x)\n",
    "\n",
    "first_blocks = Model(resnet50.input, first_layer.output)\n",
    "last_block = Model(inputs, x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_3\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 7, 7, 2048)] 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_conv (Conv2D)    (None, 7, 7, 512)    1049088     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block3_1_conv[1][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_relu (Activation (None, 7, 7, 512)    0           conv5_block3_1_bn[1][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_conv (Conv2D)    (None, 7, 7, 512)    2359808     conv5_block3_1_relu[1][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block3_2_conv[1][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_relu (Activation (None, 7, 7, 512)    0           conv5_block3_2_bn[1][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_3_conv (Conv2D)    (None, 7, 7, 2048)   1050624     conv5_block3_2_relu[1][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_3_bn (BatchNormali (None, 7, 7, 2048)   8192        conv5_block3_3_conv[1][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_add (Add)          (None, 7, 7, 2048)   0           input_2[0][0]                    \n",
      "                                                                 conv5_block3_3_bn[1][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_out (Activation)   (None, 7, 7, 2048)   0           conv5_block3_add[1][0]           \n",
      "__________________________________________________________________________________________________\n",
      "avg_pool (GlobalAveragePooling2 (None, 2048)         0           conv5_block3_out[1][0]           \n",
      "==================================================================================================\n",
      "Total params: 4,471,808\n",
      "Trainable params: 4,465,664\n",
      "Non-trainable params: 6,144\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def load_resnet50(path=\"resnet50_last_block.hdf5\"):\n",
    "    model = load_model(path)\n",
    "    for i in range(len(model.layers)):\n",
    "        if model.layers[i].__class__.__name__ == \"BatchNormalization\":\n",
    "            model.layers[i].trainable = False\n",
    "    return model\n",
    "\n",
    "last_block.summary()\n",
    "last_block.save(\"resnet50_last_block.hdf5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.constraints import MaxNorm\n",
    "\n",
    "def get_task(dropout=0.5, max_norm=0.5):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(1024, activation=\"relu\",\n",
    "                   kernel_constraint=MaxNorm(max_norm),\n",
    "                   bias_constraint=MaxNorm(max_norm)))\n",
    "    model.add(Dropout(dropout))\n",
    "    model.add(Dense(1024, activation=\"relu\",\n",
    "                   kernel_constraint=MaxNorm(max_norm),\n",
    "                   bias_constraint=MaxNorm(max_norm)))\n",
    "    model.add(Dropout(dropout))\n",
    "    model.add(Dense(31, activation=\"softmax\",\n",
    "                   kernel_constraint=MaxNorm(max_norm),\n",
    "                   bias_constraint=MaxNorm(max_norm)))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.optimizers.schedules import LearningRateSchedule\n",
    "from tensorflow.keras.optimizers import SGD\n",
    "\n",
    "class MyDecay(LearningRateSchedule):\n",
    "\n",
    "    def __init__(self, max_steps=1000, mu_0=0.01, alpha=10, beta=0.75):\n",
    "        self.mu_0 = mu_0\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "        self.max_steps = float(max_steps)\n",
    "\n",
    "    def __call__(self, step):\n",
    "        p = step / self.max_steps\n",
    "        return self.mu_0 / (1+self.alpha * p)**self.beta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.04\n",
    "momentum = 0.9\n",
    "alpha = 0.0002\n",
    "\n",
    "optimizer_task = SGD(learning_rate=MyDecay(mu_0=lr, alpha=alpha),\n",
    "                     momentum=momentum, nesterov=True)\n",
    "optimizer_enc = SGD(learning_rate=MyDecay(mu_0=lr/10., alpha=alpha),\n",
    "                    momentum=momentum, nesterov=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normal Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    }
   ],
   "source": [
    "from adapt.parameter_based import FineTuning\n",
    "\n",
    "finetunig = FineTuning(encoder=load_resnet50(),\n",
    "                         task=get_task(),\n",
    "                         optimizer=optimizer_task,\n",
    "                         optimizer_enc=optimizer_enc,\n",
    "                         loss=\"categorical_crossentropy\",\n",
    "                         metrics=[\"acc\"],\n",
    "                         copy=False,\n",
    "                         pretrain=True,\n",
    "                         pretrain__epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-14 11:01:58.742681: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.92GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-14 11:01:58.743821: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 1 with properties: \n",
      "pciBusID: 0000:0a:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.93GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-14 11:01:58.743876: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-05-14 11:01:58.743924: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10\n",
      "2021-05-14 11:01:58.743957: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10\n",
      "2021-05-14 11:01:58.743989: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10\n",
      "2021-05-14 11:01:58.744021: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10\n",
      "2021-05-14 11:01:58.744052: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10\n",
      "2021-05-14 11:01:58.744084: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-05-14 11:01:58.748609: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0, 1\n",
      "2021-05-14 11:01:58.748692: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2021-05-14 11:01:58.748706: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263]      0 1 \n",
      "2021-05-14 11:01:58.748714: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0:   N Y \n",
      "2021-05-14 11:01:58.748720: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 1:   Y N \n",
      "2021-05-14 11:01:58.750999: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11021 MB memory) -> physical GPU (device: 0, name: GeForce GTX TITAN X, pci bus id: 0000:01:00.0, compute capability: 5.2)\n",
      "2021-05-14 11:01:58.752017: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 4063 MB memory) -> physical GPU (device: 1, name: GeForce GTX TITAN X, pci bus id: 0000:0a:00.0, compute capability: 5.2)\n",
      "2021-05-14 11:02:01.088707: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-05-14 11:02:01.991597: W tensorflow/stream_executor/gpu/asm_compiler.cc:116] *** WARNING *** You are using ptxas 9.0.176, which is older than 9.2.88. ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading to incorrect results or invalid-address errors.\n",
      "\n",
      "You do not need to update to CUDA 9.2.88; cherry-picking the ptxas binary is sufficient.\n",
      "2021-05-14 11:02:02.134470: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X source shape: (2254, 7, 7, 2048)\n",
      "X target shape: (795, 7, 7, 2048)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shashemi/miniconda3/envs/tf/lib/python3.8/site-packages/sklearn/preprocessing/_encoders.py:808: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import tensorflow as tf \n",
    "\n",
    "gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)\n",
    "session = tf.compat.v1.InteractiveSession(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))\n",
    "\n",
    "X_source = first_blocks.predict(preprocess_input(Xs))\n",
    "X_target = first_blocks.predict(preprocess_input(Xt))\n",
    "\n",
    "one = OneHotEncoder(sparse=False)\n",
    "one.fit(np.array(ys).reshape(-1, 1))\n",
    "\n",
    "y_source = one.transform(np.array(ys).reshape(-1, 1))\n",
    "y_target = one.transform(np.array(yt).reshape(-1, 1))\n",
    "\n",
    "print(\"X source shape: %s\"%str(X_source.shape))\n",
    "print(\"X target shape: %s\"%str(X_target.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shashemi/miniconda3/envs/tf/lib/python3.8/site-packages/adapt/base.py:1108: UserWarning: The model has already been compiled. To perform pretraining, the model will be compiled again. Please make sure to pass the compile parameters in __init__ to avoid errors.\n",
      "  warnings.warn(\"The model has already been compiled. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "71/71 [==============================] - 2s 31ms/step - loss: 1.8341 - acc: 0.4855 - val_loss: 2.0844 - val_acc: 0.4415\n",
      "Epoch 2/5\n",
      "71/71 [==============================] - 2s 25ms/step - loss: 1.6973 - acc: 0.5189 - val_loss: 2.1329 - val_acc: 0.4591\n",
      "Epoch 3/5\n",
      "71/71 [==============================] - 2s 23ms/step - loss: 1.6789 - acc: 0.5396 - val_loss: 2.0786 - val_acc: 0.3962\n",
      "Epoch 4/5\n",
      "71/71 [==============================] - 2s 25ms/step - loss: 1.6784 - acc: 0.5326 - val_loss: 2.1376 - val_acc: 0.4453\n",
      "Epoch 5/5\n",
      "71/71 [==============================] - 2s 25ms/step - loss: 1.7345 - acc: 0.5132 - val_loss: 2.0120 - val_acc: 0.4667\n",
      "Epoch 1/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 1.5667 - acc: 0.5704 - val_loss: 2.1505 - val_acc: 0.4893\n",
      "Epoch 2/50\n",
      "71/71 [==============================] - 2s 33ms/step - loss: 1.2383 - acc: 0.6730 - val_loss: 1.8533 - val_acc: 0.5371\n",
      "Epoch 3/50\n",
      "71/71 [==============================] - 2s 32ms/step - loss: 0.9763 - acc: 0.7416 - val_loss: 1.9650 - val_acc: 0.5182\n",
      "Epoch 4/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 0.8807 - acc: 0.7619 - val_loss: 2.0676 - val_acc: 0.5296\n",
      "Epoch 5/50\n",
      "71/71 [==============================] - 3s 42ms/step - loss: 0.8386 - acc: 0.7835 - val_loss: 1.9108 - val_acc: 0.5157\n",
      "Epoch 6/50\n",
      "71/71 [==============================] - 2s 35ms/step - loss: 0.7369 - acc: 0.8112 - val_loss: 1.9921 - val_acc: 0.5459\n",
      "Epoch 7/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 0.5365 - acc: 0.8614 - val_loss: 2.2441 - val_acc: 0.5082\n",
      "Epoch 8/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.5106 - acc: 0.8666 - val_loss: 2.1089 - val_acc: 0.5220\n",
      "Epoch 9/50\n",
      "71/71 [==============================] - 3s 36ms/step - loss: 0.4369 - acc: 0.8869 - val_loss: 2.1065 - val_acc: 0.5459\n",
      "Epoch 10/50\n",
      "71/71 [==============================] - 3s 36ms/step - loss: 0.4204 - acc: 0.8988 - val_loss: 2.4316 - val_acc: 0.4478\n",
      "Epoch 11/50\n",
      "71/71 [==============================] - 2s 34ms/step - loss: 0.3477 - acc: 0.9098 - val_loss: 2.1849 - val_acc: 0.5358\n",
      "Epoch 12/50\n",
      "71/71 [==============================] - 3s 40ms/step - loss: 0.2530 - acc: 0.9397 - val_loss: 2.4108 - val_acc: 0.5270\n",
      "Epoch 13/50\n",
      "71/71 [==============================] - 3s 37ms/step - loss: 0.2921 - acc: 0.9265 - val_loss: 2.3467 - val_acc: 0.5358\n",
      "Epoch 14/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.3175 - acc: 0.9217 - val_loss: 2.4255 - val_acc: 0.5270\n",
      "Epoch 15/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 0.2820 - acc: 0.9340 - val_loss: 2.3847 - val_acc: 0.4918\n",
      "Epoch 16/50\n",
      "71/71 [==============================] - 3s 45ms/step - loss: 0.2609 - acc: 0.9327 - val_loss: 2.6925 - val_acc: 0.5069\n",
      "Epoch 17/50\n",
      "71/71 [==============================] - 3s 40ms/step - loss: 0.3331 - acc: 0.9278 - val_loss: 2.4056 - val_acc: 0.5031\n",
      "Epoch 18/50\n",
      "71/71 [==============================] - 3s 41ms/step - loss: 0.3030 - acc: 0.9366 - val_loss: 2.3084 - val_acc: 0.5233\n",
      "Epoch 19/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.1907 - acc: 0.9551 - val_loss: 2.2059 - val_acc: 0.5346\n",
      "Epoch 20/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.1933 - acc: 0.9569 - val_loss: 2.4568 - val_acc: 0.5346\n",
      "Epoch 21/50\n",
      "71/71 [==============================] - 3s 40ms/step - loss: 0.1933 - acc: 0.9560 - val_loss: 2.7118 - val_acc: 0.4881\n",
      "Epoch 22/50\n",
      "71/71 [==============================] - 3s 36ms/step - loss: 0.1009 - acc: 0.9740 - val_loss: 2.8982 - val_acc: 0.5283\n",
      "Epoch 23/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 0.0973 - acc: 0.9798 - val_loss: 3.1799 - val_acc: 0.5019\n",
      "Epoch 24/50\n",
      "71/71 [==============================] - 3s 42ms/step - loss: 0.1490 - acc: 0.9639 - val_loss: 2.9741 - val_acc: 0.4692\n",
      "Epoch 25/50\n",
      "71/71 [==============================] - 3s 40ms/step - loss: 0.3151 - acc: 0.9459 - val_loss: 2.8652 - val_acc: 0.4843\n",
      "Epoch 26/50\n",
      "71/71 [==============================] - 3s 40ms/step - loss: 0.2561 - acc: 0.9340 - val_loss: 2.8930 - val_acc: 0.5182\n",
      "Epoch 27/50\n",
      "71/71 [==============================] - 2s 34ms/step - loss: 0.1205 - acc: 0.9718 - val_loss: 2.6765 - val_acc: 0.5447\n",
      "Epoch 28/50\n",
      "71/71 [==============================] - 2s 34ms/step - loss: 0.1000 - acc: 0.9767 - val_loss: 3.0513 - val_acc: 0.5447\n",
      "Epoch 29/50\n",
      "71/71 [==============================] - 3s 37ms/step - loss: 0.1284 - acc: 0.9710 - val_loss: 3.1030 - val_acc: 0.4868\n",
      "Epoch 30/50\n",
      "71/71 [==============================] - 3s 41ms/step - loss: 0.0810 - acc: 0.9824 - val_loss: 2.7055 - val_acc: 0.5396\n",
      "Epoch 31/50\n",
      "71/71 [==============================] - 3s 36ms/step - loss: 0.0400 - acc: 0.9912 - val_loss: 3.1704 - val_acc: 0.5258\n",
      "Epoch 32/50\n",
      "71/71 [==============================] - 2s 34ms/step - loss: 0.0435 - acc: 0.9886 - val_loss: 2.7496 - val_acc: 0.5635\n",
      "Epoch 33/50\n",
      "71/71 [==============================] - 3s 40ms/step - loss: 0.1582 - acc: 0.9696 - val_loss: 2.6698 - val_acc: 0.5082\n",
      "Epoch 34/50\n",
      "71/71 [==============================] - 3s 42ms/step - loss: 0.1580 - acc: 0.9599 - val_loss: 2.6486 - val_acc: 0.5082\n",
      "Epoch 35/50\n",
      "71/71 [==============================] - 3s 36ms/step - loss: 0.0934 - acc: 0.9802 - val_loss: 3.4288 - val_acc: 0.5031\n",
      "Epoch 36/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 0.2083 - acc: 0.9582 - val_loss: 2.6580 - val_acc: 0.5358\n",
      "Epoch 37/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 0.2549 - acc: 0.9472 - val_loss: 3.1065 - val_acc: 0.5358\n",
      "Epoch 38/50\n",
      "71/71 [==============================] - 3s 37ms/step - loss: 0.1472 - acc: 0.9679 - val_loss: 2.4708 - val_acc: 0.5472\n",
      "Epoch 39/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.0755 - acc: 0.9859 - val_loss: 3.0871 - val_acc: 0.5308\n",
      "Epoch 40/50\n",
      "71/71 [==============================] - 2s 35ms/step - loss: 0.1422 - acc: 0.9740 - val_loss: 2.5991 - val_acc: 0.5321\n",
      "Epoch 41/50\n",
      "71/71 [==============================] - 2s 32ms/step - loss: 0.0768 - acc: 0.9837 - val_loss: 3.0134 - val_acc: 0.5270\n",
      "Epoch 42/50\n",
      "71/71 [==============================] - 3s 37ms/step - loss: 0.1279 - acc: 0.9754 - val_loss: 2.6041 - val_acc: 0.5321\n",
      "Epoch 43/50\n",
      "71/71 [==============================] - 3s 38ms/step - loss: 0.1088 - acc: 0.9754 - val_loss: 3.0312 - val_acc: 0.5145\n",
      "Epoch 44/50\n",
      "71/71 [==============================] - 2s 35ms/step - loss: 0.0512 - acc: 0.9894 - val_loss: 2.7629 - val_acc: 0.5447\n",
      "Epoch 45/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.0280 - acc: 0.9947 - val_loss: 4.0369 - val_acc: 0.5258\n",
      "Epoch 46/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.0386 - acc: 0.9934 - val_loss: 3.1922 - val_acc: 0.5308\n",
      "Epoch 47/50\n",
      "71/71 [==============================] - 2s 35ms/step - loss: 0.1014 - acc: 0.9828 - val_loss: 3.1563 - val_acc: 0.5321\n",
      "Epoch 48/50\n",
      "71/71 [==============================] - 2s 35ms/step - loss: 0.1209 - acc: 0.9758 - val_loss: 3.2513 - val_acc: 0.4579\n",
      "Epoch 49/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.0677 - acc: 0.9850 - val_loss: 3.0336 - val_acc: 0.5019\n",
      "Epoch 50/50\n",
      "71/71 [==============================] - 3s 39ms/step - loss: 0.0899 - acc: 0.9833 - val_loss: 3.0402 - val_acc: 0.5296\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<adapt.parameter_based._finetuning.FineTuning at 0x7ff7643edd60>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finetunig.fit(X_source, y_source, epochs=50, batch_size=32, validation_data=(X_target, y_target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 45308 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "import numpy as np\n",
    "\n",
    "acc = finetunig.history.history[\"acc\"]\n",
    "val_acc = finetunig.history.history[\"val_acc\"]\n",
    "\n",
    "plt.plot(acc, label=\"Train acc - final value: %.3f\"%acc[-1])\n",
    "plt.plot(val_acc, label=\"Test acc - final value: %.3f\"%val_acc[-1])\n",
    "plt.legend(); plt.xlabel(\"Epochs\"); plt.ylabel(\"Acc\"); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 89356 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "Xs_enc = finetunig.transform(X_source)\n",
    "Xt_enc = finetunig.transform(X_target)\n",
    "\n",
    "np.random.seed(0)\n",
    "X_ = np.concatenate((Xs_enc, Xt_enc))\n",
    "X_tsne = TSNE(2).fit_transform(X_)\n",
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(X_tsne[:len(X_source), 0], X_tsne[:len(X_source), 1], '.', label=\"source\")\n",
    "plt.plot(X_tsne[len(X_source):, 0], X_tsne[len(X_source):, 1], '.', label=\"target\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Encoded Space tSNE for the Source Only model\")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adapt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We shuffle the data, it may not be needed but we want to avoid classes being not represented\n",
    "# in some batch due to the fact that the datasets are sorted by classes.\n",
    "# np.random.seed(0)\n",
    "# shuffle_src = np.random.choice(len(Xs), len(Xs), replace=False)\n",
    "# shuffle_tgt = np.random.choice(len(Xt), len(Xt), replace=False)\n",
    "\n",
    "# Xs = Xs[shuffle_src]\n",
    "# ys_lab = ys_lab[shuffle_src]\n",
    "# Xt = Xt[shuffle_tgt]\n",
    "# yt_lab = yt_lab[shuffle_tgt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    }
   ],
   "source": [
    "from adapt.feature_based import MDD\n",
    "from adapt.utils import UpdateLambda\n",
    "import tensorflow as tf \n",
    "\n",
    "np.random.seed(123)\n",
    "tf.random.set_seed(123)\n",
    "\n",
    "lr = 0.04\n",
    "momentum = 0.9\n",
    "alpha = 0.0002\n",
    "\n",
    "encoder = load_resnet50()\n",
    "task = get_task()\n",
    "\n",
    "optimizer_task = SGD(learning_rate=MyDecay(mu_0=lr, alpha=alpha),\n",
    "                     momentum=momentum, nesterov=True)\n",
    "optimizer_enc = SGD(learning_rate=MyDecay(mu_0=lr/10., alpha=alpha),\n",
    "                    momentum=momentum, nesterov=True)\n",
    "optimizer_disc = SGD(learning_rate=MyDecay(mu_0=lr/10., alpha=alpha))\n",
    "\n",
    "\n",
    "mdd = MDD(encoder, task,\n",
    "            loss=\"categorical_crossentropy\",\n",
    "            metrics=[\"acc\"],\n",
    "            copy=False,\n",
    "            lambda_=tf.Variable(0.),\n",
    "            gamma=2.,\n",
    "            optimizer=optimizer_task,\n",
    "            optimizer_enc=optimizer_enc,\n",
    "            optimizer_disc=optimizer_disc,\n",
    "            callbacks=[UpdateLambda(lambda_max=0.1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer mdd_1 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "Epoch 1/50\n",
      "71/71 [==============================] - 4s 54ms/step - loss: 2.8755 - acc: 0.2108 - disc_loss: 6.8939 - val_loss: 2.3044 - val_acc: 0.3686\n",
      "Epoch 2/50\n",
      "71/71 [==============================] - 4s 61ms/step - loss: 1.6812 - acc: 0.5260 - disc_loss: 6.6367 - val_loss: 1.8391 - val_acc: 0.5208\n",
      "Epoch 3/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 1.2456 - acc: 0.6567 - disc_loss: 6.2507 - val_loss: 2.0318 - val_acc: 0.5208\n",
      "Epoch 4/50\n",
      "71/71 [==============================] - 4s 62ms/step - loss: 1.0195 - acc: 0.7205 - disc_loss: 5.6312 - val_loss: 2.1595 - val_acc: 0.4818\n",
      "Epoch 5/50\n",
      "71/71 [==============================] - 4s 56ms/step - loss: 0.8656 - acc: 0.7658 - disc_loss: 4.9793 - val_loss: 2.0189 - val_acc: 0.4981\n",
      "Epoch 6/50\n",
      "71/71 [==============================] - 4s 56ms/step - loss: 0.7520 - acc: 0.7909 - disc_loss: 4.3038 - val_loss: 2.0078 - val_acc: 0.4730\n",
      "Epoch 7/50\n",
      "71/71 [==============================] - 4s 61ms/step - loss: 0.6110 - acc: 0.8393 - disc_loss: 3.8673 - val_loss: 2.0799 - val_acc: 0.5031\n",
      "Epoch 8/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 0.4696 - acc: 0.8724 - disc_loss: 3.3101 - val_loss: 1.7728 - val_acc: 0.5925\n",
      "Epoch 9/50\n",
      "71/71 [==============================] - 4s 61ms/step - loss: 0.4419 - acc: 0.8873 - disc_loss: 3.0773 - val_loss: 2.0565 - val_acc: 0.4956\n",
      "Epoch 10/50\n",
      "71/71 [==============================] - 4s 60ms/step - loss: 0.3757 - acc: 0.9058 - disc_loss: 2.8416 - val_loss: 2.3294 - val_acc: 0.5107\n",
      "Epoch 11/50\n",
      "71/71 [==============================] - 4s 56ms/step - loss: 0.2926 - acc: 0.9239 - disc_loss: 2.7525 - val_loss: 1.9341 - val_acc: 0.5862\n",
      "Epoch 12/50\n",
      "71/71 [==============================] - 4s 56ms/step - loss: 0.2152 - acc: 0.9428 - disc_loss: 2.7566 - val_loss: 2.2483 - val_acc: 0.5761\n",
      "Epoch 13/50\n",
      "71/71 [==============================] - 4s 62ms/step - loss: 0.1624 - acc: 0.9648 - disc_loss: 2.9224 - val_loss: 2.6861 - val_acc: 0.5874\n",
      "Epoch 14/50\n",
      "71/71 [==============================] - 4s 60ms/step - loss: 0.1547 - acc: 0.9665 - disc_loss: 3.1766 - val_loss: 2.9834 - val_acc: 0.5874\n",
      "Epoch 15/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 0.1280 - acc: 0.9740 - disc_loss: 3.4739 - val_loss: 2.9884 - val_acc: 0.6075\n",
      "Epoch 16/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 0.1501 - acc: 0.9674 - disc_loss: 3.7624 - val_loss: 3.6081 - val_acc: 0.5648\n",
      "Epoch 17/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.1791 - acc: 0.9604 - disc_loss: 3.6178 - val_loss: 3.0777 - val_acc: 0.5987\n",
      "Epoch 18/50\n",
      "71/71 [==============================] - 4s 61ms/step - loss: 0.2074 - acc: 0.9551 - disc_loss: 3.6645 - val_loss: 4.1626 - val_acc: 0.5522\n",
      "Epoch 19/50\n",
      "71/71 [==============================] - 4s 57ms/step - loss: 0.2284 - acc: 0.9555 - disc_loss: 3.4961 - val_loss: 4.9788 - val_acc: 0.5396\n",
      "Epoch 20/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 0.2392 - acc: 0.9498 - disc_loss: 3.3149 - val_loss: 4.0270 - val_acc: 0.6176\n",
      "Epoch 21/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 0.2286 - acc: 0.9604 - disc_loss: 3.3962 - val_loss: 4.3935 - val_acc: 0.5711\n",
      "Epoch 22/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.1820 - acc: 0.9604 - disc_loss: 3.4481 - val_loss: 4.7817 - val_acc: 0.5987\n",
      "Epoch 23/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.1604 - acc: 0.9670 - disc_loss: 3.4992 - val_loss: 5.1246 - val_acc: 0.5824\n",
      "Epoch 24/50\n",
      "71/71 [==============================] - 5s 64ms/step - loss: 0.1728 - acc: 0.9665 - disc_loss: 3.6277 - val_loss: 5.7745 - val_acc: 0.5686\n",
      "Epoch 25/50\n",
      "71/71 [==============================] - 4s 53ms/step - loss: 0.3831 - acc: 0.9159 - disc_loss: 3.6940 - val_loss: 3.8723 - val_acc: 0.6214\n",
      "Epoch 26/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.1994 - acc: 0.9573 - disc_loss: 3.5767 - val_loss: 5.5462 - val_acc: 0.5522\n",
      "Epoch 27/50\n",
      "71/71 [==============================] - 4s 57ms/step - loss: 0.1861 - acc: 0.9639 - disc_loss: 3.6581 - val_loss: 4.6228 - val_acc: 0.6013\n",
      "Epoch 28/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.2846 - acc: 0.9379 - disc_loss: 3.6176 - val_loss: 4.3204 - val_acc: 0.6214\n",
      "Epoch 29/50\n",
      "71/71 [==============================] - 4s 57ms/step - loss: 0.2285 - acc: 0.9525 - disc_loss: 3.5272 - val_loss: 4.9658 - val_acc: 0.6176\n",
      "Epoch 30/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 0.1397 - acc: 0.9710 - disc_loss: 3.5457 - val_loss: 6.4727 - val_acc: 0.6063\n",
      "Epoch 31/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.3276 - acc: 0.9450 - disc_loss: 3.9350 - val_loss: 6.0114 - val_acc: 0.5623\n",
      "Epoch 32/50\n",
      "71/71 [==============================] - 5s 64ms/step - loss: 0.3092 - acc: 0.9357 - disc_loss: 3.4947 - val_loss: 4.7175 - val_acc: 0.6289\n",
      "Epoch 33/50\n",
      "71/71 [==============================] - 4s 53ms/step - loss: 0.2839 - acc: 0.9450 - disc_loss: 3.4465 - val_loss: 4.6049 - val_acc: 0.6126\n",
      "Epoch 34/50\n",
      "71/71 [==============================] - 3s 46ms/step - loss: 0.1951 - acc: 0.9635 - disc_loss: 3.5649 - val_loss: 6.4033 - val_acc: 0.5623\n",
      "Epoch 35/50\n",
      "71/71 [==============================] - 4s 50ms/step - loss: 0.4804 - acc: 0.8904 - disc_loss: 3.8366 - val_loss: 5.5151 - val_acc: 0.5560\n",
      "Epoch 36/50\n",
      "71/71 [==============================] - 3s 48ms/step - loss: 0.2815 - acc: 0.9362 - disc_loss: 3.4471 - val_loss: 4.7744 - val_acc: 0.6226\n",
      "Epoch 37/50\n",
      "71/71 [==============================] - 4s 51ms/step - loss: 0.1950 - acc: 0.9591 - disc_loss: 3.3060 - val_loss: 5.5707 - val_acc: 0.5987\n",
      "Epoch 38/50\n",
      "71/71 [==============================] - 4s 52ms/step - loss: 0.1268 - acc: 0.9837 - disc_loss: 3.4041 - val_loss: 5.6056 - val_acc: 0.6201\n",
      "Epoch 39/50\n",
      "71/71 [==============================] - 4s 55ms/step - loss: 0.2663 - acc: 0.9586 - disc_loss: 3.7014 - val_loss: 5.5689 - val_acc: 0.6038\n",
      "Epoch 40/50\n",
      "71/71 [==============================] - 4s 61ms/step - loss: 0.1119 - acc: 0.9749 - disc_loss: 3.4972 - val_loss: 6.6409 - val_acc: 0.6126\n",
      "Epoch 41/50\n",
      "71/71 [==============================] - 4s 57ms/step - loss: 0.1701 - acc: 0.9621 - disc_loss: 4.1863 - val_loss: 7.7062 - val_acc: 0.6428\n",
      "Epoch 42/50\n",
      "71/71 [==============================] - 4s 61ms/step - loss: 0.1188 - acc: 0.9771 - disc_loss: 3.8941 - val_loss: 7.5871 - val_acc: 0.6176\n",
      "Epoch 43/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.5638 - acc: 0.9265 - disc_loss: 4.4041 - val_loss: 6.5325 - val_acc: 0.5459\n",
      "Epoch 44/50\n",
      "71/71 [==============================] - 4s 60ms/step - loss: 0.4141 - acc: 0.9305 - disc_loss: 3.7344 - val_loss: 6.1255 - val_acc: 0.6239\n",
      "Epoch 45/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.2632 - acc: 0.9604 - disc_loss: 3.4305 - val_loss: 6.7876 - val_acc: 0.6277\n",
      "Epoch 46/50\n",
      "71/71 [==============================] - 4s 60ms/step - loss: 0.2809 - acc: 0.9498 - disc_loss: 3.4746 - val_loss: 5.7009 - val_acc: 0.6528\n",
      "Epoch 47/50\n",
      "71/71 [==============================] - 4s 58ms/step - loss: 0.3646 - acc: 0.9366 - disc_loss: 3.4981 - val_loss: 5.4116 - val_acc: 0.6491\n",
      "Epoch 48/50\n",
      "71/71 [==============================] - 4s 59ms/step - loss: 0.1865 - acc: 0.9661 - disc_loss: 3.1507 - val_loss: 6.9634 - val_acc: 0.6415\n",
      "Epoch 49/50\n",
      "71/71 [==============================] - 4s 56ms/step - loss: 0.1857 - acc: 0.9665 - disc_loss: 3.1067 - val_loss: 8.2365 - val_acc: 0.6377\n",
      "Epoch 50/50\n",
      "71/71 [==============================] - 4s 56ms/step - loss: 0.0918 - acc: 0.9828 - disc_loss: 3.3619 - val_loss: 8.9936 - val_acc: 0.6440\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<adapt.feature_based._mdd.MDD at 0x7ff93a75bf10>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdd.fit(X=X_source[:-1], y=y_source[:-1], Xt=X_target, epochs=50, batch_size=32, validation_data=(X_target, y_target))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 48072 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = mdd.history.history[\"acc\"]\n",
    "val_acc = mdd.history.history[\"val_acc\"]\n",
    "plt.plot(acc, label=\"Train acc - final value: %.3f\"%acc[-1])\n",
    "plt.plot(val_acc, label=\"Test acc - final value: %.3f\"%val_acc[-1])\n",
    "plt.legend(); plt.xlabel(\"Epochs\"); plt.ylabel(\"Acc\"); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 88064 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "Xs_enc = mdd.transform(X_source)\n",
    "Xt_enc = mdd.transform(X_target)\n",
    "\n",
    "np.random.seed(0)\n",
    "X_ = np.concatenate((Xs_enc, Xt_enc))\n",
    "X_tsne = TSNE(2).fit_transform(X_)\n",
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(X_tsne[:len(X_source), 0], X_tsne[:len(X_source), 1], '.', label=\"source\")\n",
    "plt.plot(X_tsne[len(X_source):, 0], X_tsne[len(X_source):, 1], '.', label=\"target\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Encoded Space tSNE for the MDD model\")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CherkNevis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dt_from_tensorflow(path):\n",
    "    validation_dataset =tf.keras.preprocessing.image_dataset_from_directory(path,\n",
    "                                              shuffle=True,\n",
    "                                              batch_size=32,\n",
    "                                              image_size=(224,224))\n",
    "    return validation_dataset\n",
    "\n",
    "def load_image_from_tensorflow(path):\n",
    "    validation_dataset =tf.keras.preprocessing.image_dataset_from_directory(path,\n",
    "                                              shuffle=False,\n",
    "                                              batch_size=32,\n",
    "                                              image_size=(224,224))\n",
    "    for image, _ in validation_dataset.take(1):\n",
    "        img = image\n",
    "    return img\n",
    "\n",
    "def load_image_from_pil(path):\n",
    "    img = Image.open(path)\n",
    "    img = img.resize((224, 224),Image.BILINEAR)\n",
    "    img = np.expand_dims(img, axis=0)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 224, 224, 3), (None,)), types: (tf.float32, tf.int32)>\n",
      "Found 2 files belonging to 2 classes.\n",
      "[255. 255. 255.] [168.79688 158.79688 146.79688]\n",
      "[1 0]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[255. 255. 255.] [168.79688 158.79688 146.79688]\n",
      "[1 0]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[255. 255. 255.] [168.79688 158.79688 146.79688]\n",
      "[1 0]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[255. 255. 255.] [168.79688 158.79688 146.79688]\n",
      "[1 0]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[168.79688 158.79688 146.79688] [255. 255. 255.]\n",
      "[0 1]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[168.79688 158.79688 146.79688] [255. 255. 255.]\n",
      "[0 1]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[168.79688 158.79688 146.79688] [255. 255. 255.]\n",
      "[0 1]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[168.79688 158.79688 146.79688] [255. 255. 255.]\n",
      "[0 1]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[255. 255. 255.] [168.79688 158.79688 146.79688]\n",
      "[1 0]\n",
      "Found 2 files belonging to 2 classes.\n",
      "[255. 255. 255.] [168.79688 158.79688 146.79688]\n",
      "[1 0]\n",
      "0:00:01.247278\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "import datetime\n",
    "print(d)\n",
    "\n",
    "s = datetime.datetime.now()\n",
    "for i in range(10):\n",
    "    d = load_dt_from_tensorflow('datasets/temp')\n",
    "    xy = [(x, y) for x, y in d]\n",
    "    xx = np.concatenate([x for x, y in xy], axis=0)\n",
    "    yy = np.concatenate([y for x, y in xy], axis=0)\n",
    "\n",
    "    print(xx[0][0][0], xx[1][0][0])\n",
    "    print(yy)\n",
    "\n",
    "e = datetime.datetime.now()\n",
    "print(e-s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([168.79688 158.79688 146.79688], shape=(3,), dtype=float32) tf.Tensor([255. 255. 255.], shape=(3,), dtype=float32)\n",
      "[<tf.Tensor: shape=(), dtype=int32, numpy=1>, <tf.Tensor: shape=(), dtype=int32, numpy=0>]\n",
      "0:00:00.197981\n"
     ]
    }
   ],
   "source": [
    "s = datetime.datetime.now()\n",
    "dd = d.unbatch()\n",
    "xy = dd.map(lambda x, y: (x,y))\n",
    "xx = list(xy.map(lambda x, y: x))\n",
    "yy = list(xy.map(lambda x, y: y))\n",
    "print(xx[0][0][0], xx[1][0][0])\n",
    "print(yy)\n",
    "e = datetime.datetime.now()\n",
    "print(e-s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1 files belonging to 1 classes.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_12245/1478134846.py:12: DeprecationWarning: BILINEAR is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BILINEAR instead.\n",
      "  img = img.resize((224, 224),Image.BILINEAR)\n"
     ]
    }
   ],
   "source": [
    "i1 = load_image_from_tensorflow('datasets/temp')\n",
    "i2 = load_image_from_pil('datasets/temp/saba/frame_0001.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([36.0625 37.0625 32.0625], shape=(3,), dtype=float32)\n",
      "[36 36 34]\n"
     ]
    }
   ],
   "source": [
    "idx = 150\n",
    "print(i1[0][idx][idx])\n",
    "print(i2[0][idx][idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'tensorflow.python.framework.ops.EagerTensor' object has no attribute 'astype'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[55], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m p1 \u001b[39m=\u001b[39m first_blocks\u001b[39m.\u001b[39mpredict(preprocess_input(i1\u001b[39m.\u001b[39;49mastype(np\u001b[39m.\u001b[39mint32)))\n\u001b[1;32m      2\u001b[0m p2 \u001b[39m=\u001b[39m first_blocks\u001b[39m.\u001b[39mpredict(preprocess_input(i2))\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'tensorflow.python.framework.ops.EagerTensor' object has no attribute 'astype'"
     ]
    }
   ],
   "source": [
    "p1 = first_blocks.predict(preprocess_input(i1.astype(np.int32)))\n",
    "p2 = first_blocks.predict(preprocess_input(i2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 7, 7, 2048)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.01094218 0.         0.         0.         0.\n",
      " 0.         0.08283848 0.         0.        ]\n",
      "[0.         0.         0.         0.         0.         0.\n",
      " 0.         0.10250913 0.         0.        ]\n"
     ]
    }
   ],
   "source": [
    "idx, idx2 = 1, 1045\n",
    "print(p1[0][idx][idx][0:10])\n",
    "print(p2[0][idx][idx][0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer mdd is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-13 09:48:13.580128: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer flatten is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "1/1 [==============================] - 0s 1ms/step - loss: 0.1545 - acc: 0.9100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.15446723997592926"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from adapt.utils import make_classification_da\n",
    "from adapt.feature_based import MDD\n",
    "Xs, ys, Xt, yt = make_classification_da()\n",
    "model = MDD(lambda_=0.1, gamma=4., Xt=Xt, metrics=[\"acc\"], random_state=0)\n",
    "model.fit(Xs, ys, epochs=100, verbose=0)\n",
    "model.score(Xt, yt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit transform...\n",
      "Previous covariance difference: 0.013181\n",
      "New covariance difference: 0.000004\n",
      "Fit Estimator...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.86"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import RidgeClassifier\n",
    "from adapt.feature_based import CORAL\n",
    "\n",
    "model = CORAL(RidgeClassifier(), Xt=Xt, random_state=0)\n",
    "model.fit(Xs, ys)\n",
    "model.score(Xt, yt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'X_source' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[18], line 30\u001b[0m\n\u001b[1;32m     19\u001b[0m optimizer_disc \u001b[39m=\u001b[39m SGD(learning_rate\u001b[39m=\u001b[39mMyDecay(mu_0\u001b[39m=\u001b[39mlr\u001b[39m/\u001b[39m\u001b[39m10.\u001b[39m, alpha\u001b[39m=\u001b[39malpha))\n\u001b[1;32m     22\u001b[0m mdd \u001b[39m=\u001b[39m CORAL(encoder, task,\n\u001b[1;32m     23\u001b[0m             loss\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mcategorical_crossentropy\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m     24\u001b[0m             metrics\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39macc\u001b[39m\u001b[39m\"\u001b[39m],\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     27\u001b[0m             optimizer\u001b[39m=\u001b[39moptimizer_task,\n\u001b[1;32m     28\u001b[0m             callbacks\u001b[39m=\u001b[39m[UpdateLambda(lambda_max\u001b[39m=\u001b[39m\u001b[39m0.1\u001b[39m)])\n\u001b[0;32m---> 30\u001b[0m model\u001b[39m.\u001b[39mfit(X_source, y_source)\n\u001b[1;32m     31\u001b[0m model\u001b[39m.\u001b[39mscore(X_target, y_target)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X_source' is not defined"
     ]
    }
   ],
   "source": [
    "from adapt.feature_based import CORAL\n",
    "from adapt.utils import UpdateLambda\n",
    "import tensorflow as tf \n",
    "\n",
    "np.random.seed(123)\n",
    "tf.random.set_seed(123)\n",
    "\n",
    "lr = 0.04\n",
    "momentum = 0.9\n",
    "alpha = 0.0002\n",
    "\n",
    "encoder = load_resnet50()\n",
    "task = get_task()\n",
    "\n",
    "optimizer_task = SGD(learning_rate=MyDecay(mu_0=lr, alpha=alpha),\n",
    "                     momentum=momentum, nesterov=True)\n",
    "optimizer_enc = SGD(learning_rate=MyDecay(mu_0=lr/10., alpha=alpha),\n",
    "                    momentum=momentum, nesterov=True)\n",
    "optimizer_disc = SGD(learning_rate=MyDecay(mu_0=lr/10., alpha=alpha))\n",
    "\n",
    "\n",
    "mdd = CORAL(encoder, task,\n",
    "            loss=\"categorical_crossentropy\",\n",
    "            metrics=[\"acc\"],\n",
    "            copy=False,\n",
    "            lambda_=tf.Variable(0.),\n",
    "            optimizer=optimizer_task,\n",
    "            callbacks=[UpdateLambda(lambda_max=0.1)])\n",
    "\n",
    "model.fit(X_source, y_source)\n",
    "model.score(X_target, y_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35336 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot([1,2,3,4], label=\"Train acc - final value: %.3f\"%1)\n",
    "plt.plot([3,4,10, -1], label=\"Test acc - final value: %.3f\"%3)\n",
    "plt.legend()\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Acc\")\n",
    "plt.savefig('image.png') "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow.keras.applications.resnet50 import ResNet50\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.constraints import MaxNorm\n",
    "from tensorflow.keras.optimizers.schedules import LearningRateSchedule\n",
    "from tensorflow.keras.applications.resnet50 import preprocess_input\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "\n",
    "def get_resnet():\n",
    "    # if you want to download weights, remove weights param in ResNet40 and remove this line\n",
    "    WEIGHTS_PATH = 'model-weights/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'\n",
    "\n",
    "    resnet50 = ResNet50(include_top=False, input_shape=(224, 224, 3), pooling=\"avg\", weights=WEIGHTS_PATH, classes=31)\n",
    "\n",
    "    first_layer = resnet50.get_layer('conv5_block2_out')\n",
    "    inputs = Input(first_layer.output_shape[1:])\n",
    "\n",
    "    for layer in resnet50.layers[resnet50.layers.index(first_layer)+1:]:\n",
    "        if layer.name == \"conv5_block3_1_conv\":\n",
    "            x = layer(inputs)\n",
    "        elif layer.name == \"conv5_block3_add\":\n",
    "            x = layer([inputs, x])\n",
    "        else:\n",
    "            x = layer(x)\n",
    "\n",
    "    first_blocks = Model(resnet50.input, first_layer.output)\n",
    "    last_block = Model(inputs, x)\n",
    "\n",
    "\n",
    "    def load_resnet50(path=\"model-weights/resnet50_last_block.hdf5\"):\n",
    "        model = load_model(path)\n",
    "        for i in range(len(model.layers)):\n",
    "            if model.layers[i].__class__.__name__ == \"BatchNormalization\":\n",
    "                model.layers[i].trainable = False\n",
    "        return model\n",
    "\n",
    "    # last_block.summary()\n",
    "    last_block.save(\"model-weights/resnet50_last_block.hdf5\")\n",
    "\n",
    "    return first_blocks, last_block, load_resnet50\n",
    "\n",
    "\n",
    "def get_task(dropout=0.5, max_norm=0.5):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(1024, activation=\"relu\",\n",
    "                kernel_constraint=MaxNorm(max_norm),\n",
    "                bias_constraint=MaxNorm(max_norm)))\n",
    "    model.add(Dropout(dropout))\n",
    "    model.add(Dense(1024, activation=\"relu\",\n",
    "                kernel_constraint=MaxNorm(max_norm),\n",
    "                bias_constraint=MaxNorm(max_norm)))\n",
    "    model.add(Dropout(dropout))\n",
    "    model.add(Dense(31, activation=\"softmax\",\n",
    "                kernel_constraint=MaxNorm(max_norm),\n",
    "                bias_constraint=MaxNorm(max_norm)))\n",
    "    return model\n",
    "\n",
    "\n",
    "class MyDecay(LearningRateSchedule):\n",
    "    def __init__(self, max_steps=1000, mu_0=0.01, alpha=10, beta=0.75):\n",
    "        self.mu_0 = mu_0\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "        self.max_steps = float(max_steps)\n",
    "\n",
    "    def __call__(self, step):\n",
    "        p = step / self.max_steps\n",
    "        return self.mu_0 / (1+self.alpha * p)**self.beta\n",
    "\n",
    "\n",
    "def get_input_and_target_for_head(first_blocks, Xs, ys, Xv, yv, Xt, yt):\n",
    "    X_source = first_blocks.predict(preprocess_input(Xs))\n",
    "    X_val = first_blocks.predict(preprocess_input(Xv))\n",
    "    X_target = first_blocks.predict(preprocess_input(Xt))\n",
    "\n",
    "    one = OneHotEncoder(sparse=False)\n",
    "    one.fit(np.array(ys).reshape(-1, 1))\n",
    "\n",
    "    y_source = one.transform(np.array(ys).reshape(-1, 1))\n",
    "    y_val = one.transform(np.array(yv).reshape(-1, 1))\n",
    "    y_target = one.transform(np.array(yt).reshape(-1, 1))\n",
    "\n",
    "    print(\"X source shape: %s\"%str(X_source.shape))\n",
    "    print(\"X target shape: %s\"%str(X_target.shape))\n",
    "\n",
    "    return X_source, y_source, X_val, y_val, X_target, y_target\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2817 files belonging to 31 classes.\n",
      "Using 2254 files for training.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-13 12:58:05.917553: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1\n",
      "2021-05-13 12:58:05.925685: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.92GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-13 12:58:05.926565: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 1 with properties: \n",
      "pciBusID: 0000:0a:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.93GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-13 12:58:05.926598: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-05-13 12:58:05.959468: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10\n",
      "2021-05-13 12:58:05.978114: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10\n",
      "2021-05-13 12:58:05.982830: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10\n",
      "2021-05-13 12:58:06.017680: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10\n",
      "2021-05-13 12:58:06.022805: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10\n",
      "2021-05-13 12:58:06.084876: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-05-13 12:58:06.088343: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0, 1\n",
      "2021-05-13 12:58:06.089695: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-05-13 12:58:06.103866: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 3298130000 Hz\n",
      "2021-05-13 12:58:06.104753: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5352d60 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2021-05-13 12:58:06.104782: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2021-05-13 12:58:06.264450: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x53bf060 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2021-05-13 12:58:06.264490: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): GeForce GTX TITAN X, Compute Capability 5.2\n",
      "2021-05-13 12:58:06.264502: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (1): GeForce GTX TITAN X, Compute Capability 5.2\n",
      "2021-05-13 12:58:06.266633: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties: \n",
      "pciBusID: 0000:01:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.92GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-13 12:58:06.267263: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 1 with properties: \n",
      "pciBusID: 0000:0a:00.0 name: GeForce GTX TITAN X computeCapability: 5.2\n",
      "coreClock: 1.076GHz coreCount: 24 deviceMemorySize: 11.93GiB deviceMemoryBandwidth: 313.37GiB/s\n",
      "2021-05-13 12:58:06.267315: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-05-13 12:58:06.267351: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10\n",
      "2021-05-13 12:58:06.267380: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10\n",
      "2021-05-13 12:58:06.267406: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10\n",
      "2021-05-13 12:58:06.267433: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10\n",
      "2021-05-13 12:58:06.267459: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10\n",
      "2021-05-13 12:58:06.267486: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7\n",
      "2021-05-13 12:58:06.270367: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0, 1\n",
      "2021-05-13 12:58:06.270829: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
      "2021-05-13 12:58:08.309679: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2021-05-13 12:58:08.309710: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263]      0 1 \n",
      "2021-05-13 12:58:08.309717: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0:   N Y \n",
      "2021-05-13 12:58:08.309721: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 1:   Y N \n",
      "2021-05-13 12:58:08.314308: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 11021 MB memory) -> physical GPU (device: 0, name: GeForce GTX TITAN X, pci bus id: 0000:01:00.0, compute capability: 5.2)\n",
      "2021-05-13 12:58:08.315883: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 4063 MB memory) -> physical GPU (device: 1, name: GeForce GTX TITAN X, pci bus id: 0000:0a:00.0, compute capability: 5.2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2817 files belonging to 31 classes.\n",
      "Using 563 files for validation.\n",
      "Found 795 files belonging to 31 classes.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.applications.resnet50 import preprocess_input\n",
    "from tensorflow.keras.optimizers import SGD\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import tensorflow as tf \n",
    "import matplotlib.pyplot as plt \n",
    "import numpy as np\n",
    "from adapt.feature_based import MDD\n",
    "from adapt.utils import UpdateLambda\n",
    "from data_loader import load_data, get_input_and_labels_from_batch_ds\n",
    "\n",
    "epochs=2\n",
    "batch_size=32\n",
    "\n",
    "train_ds, val_ds, target_ds = load_data(\"amazon\", \"webcam\", image_size=(224,224))\n",
    "Xs, ys = get_input_and_labels_from_batch_ds(train_ds)\n",
    "Xv, yv = get_input_and_labels_from_batch_ds(val_ds)\n",
    "Xt, yt = get_input_and_labels_from_batch_ds(target_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X source shape: (2254, 7, 7, 2048)\n",
      "X target shape: (795, 7, 7, 2048)\n",
      "WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shashemi/miniconda3/envs/tf/lib/python3.8/site-packages/sklearn/preprocessing/_encoders.py:808: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL_NAME: fine_tuning\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "fit() got multiple values for argument 'X'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[38], line 123\u001b[0m\n\u001b[1;32m    113\u001b[0m     model \u001b[39m=\u001b[39m RegularTransferNN(encoder, task,\n\u001b[1;32m    114\u001b[0m                 loss\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mcategorical_crossentropy\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m    115\u001b[0m                 metrics\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39macc\u001b[39m\u001b[39m\"\u001b[39m],\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    119\u001b[0m                 \n\u001b[1;32m    120\u001b[0m                 )\n\u001b[1;32m    122\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mMODEL_NAME:\u001b[39m\u001b[39m\"\u001b[39m, model\u001b[39m.\u001b[39mname)\n\u001b[0;32m--> 123\u001b[0m model\u001b[39m.\u001b[39;49mfit(X\u001b[39m=\u001b[39;49mX_source[:\u001b[39m-\u001b[39;49m\u001b[39m1\u001b[39;49m], y\u001b[39m=\u001b[39;49my_source[:\u001b[39m-\u001b[39;49m\u001b[39m1\u001b[39;49m], Xt\u001b[39m=\u001b[39;49mX_target, yt\u001b[39m=\u001b[39;49my_target, epochs\u001b[39m=\u001b[39;49mepochs, batch_size\u001b[39m=\u001b[39;49mbatch_size, validation_data\u001b[39m=\u001b[39;49m(X_val, y_val))\n\u001b[1;32m    125\u001b[0m acc \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mhistory\u001b[39m.\u001b[39mhistory\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39macc\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m) \u001b[39mor\u001b[39;00m model\u001b[39m.\u001b[39mhistory\u001b[39m.\u001b[39mhistory\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mdisc_acc\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m    126\u001b[0m val_acc \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mhistory\u001b[39m.\u001b[39mhistory[\u001b[39m\"\u001b[39m\u001b[39mval_acc\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[0;32m~/miniconda3/envs/tf/lib/python3.8/site-packages/adapt/parameter_based/_finetuning.py:120\u001b[0m, in \u001b[0;36mFineTuning.fit\u001b[0;34m(self, Xt, yt, **fit_params)\u001b[0m\n\u001b[1;32m    118\u001b[0m Xs \u001b[39m=\u001b[39m Xt\n\u001b[1;32m    119\u001b[0m ys \u001b[39m=\u001b[39m yt\n\u001b[0;32m--> 120\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49mfit(Xs, ys, Xt\u001b[39m=\u001b[39;49mXt, yt\u001b[39m=\u001b[39;49myt, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mfit_params)\n",
      "\u001b[0;31mTypeError\u001b[0m: fit() got multiple values for argument 'X'"
     ]
    }
   ],
   "source": [
    "from adapt.feature_based import DeepCORAL,MCD,MDD, WDGRL ,CDAN ,CCSA ,DANN,ADDA\n",
    "from adapt.parameter_based import RegularTransferNN, FineTuning\n",
    "\n",
    "first_blocks, _, load_resnet50 = get_resnet()\n",
    "X_source, y_source, X_val, y_val, X_target, y_target = get_input_and_target_for_head(first_blocks, Xs, ys, Xv, yv, Xt, yt)\n",
    "model_name =\"FineTuning\"\n",
    "\n",
    "lr = 0.04\n",
    "momentum = 0.9\n",
    "alpha = 0.0002\n",
    "\n",
    "encoder = load_resnet50()\n",
    "task = get_task()\n",
    "\n",
    "optimizer_task = SGD(learning_rate=MyDecay(mu_0=lr, alpha=alpha),\n",
    "                    momentum=momentum, nesterov=True)\n",
    "optimizer_enc = SGD(learning_rate=MyDecay(mu_0=lr/10., alpha=alpha),\n",
    "                    momentum=momentum, nesterov=True)\n",
    "optimizer_disc = SGD(learning_rate=MyDecay(mu_0=lr/10., alpha=alpha))\n",
    "\n",
    "if model_name == \"MDD\":\n",
    "    model = MDD(encoder, task,\n",
    "            loss=\"categorical_crossentropy\",\n",
    "            metrics=[\"acc\"],\n",
    "            copy=False,\n",
    "            gamma=2.,\n",
    "            lambda_=tf.Variable(0.),\n",
    "            optimizer=optimizer_task,\n",
    "            optimizer_enc=optimizer_enc,\n",
    "            optimizer_disc=optimizer_disc,\n",
    "            callbacks=[UpdateLambda(lambda_max=0.1)])\n",
    "elif model_name ==\"DeepCORAL\":\n",
    "    model = DeepCORAL(encoder, task,\n",
    "                loss=\"categorical_crossentropy\",\n",
    "                metrics=[\"acc\"],\n",
    "                copy=False,\n",
    "                lambda_=tf.Variable(0.),\n",
    "                optimizer=optimizer_task,\n",
    "                optimizer_enc=optimizer_enc,\n",
    "                optimizer_disc=optimizer_disc,\n",
    "                callbacks=[UpdateLambda(lambda_max=0.1)])\n",
    "elif model_name ==\"MCD\": \n",
    "    model = MCD(encoder, task,\n",
    "                loss=\"categorical_crossentropy\",\n",
    "                metrics=[\"acc\"],\n",
    "                copy=False,\n",
    "                optimizer=optimizer_task,\n",
    "                optimizer_enc=optimizer_enc,\n",
    "                optimizer_disc=optimizer_disc)\n",
    "elif model_name == \"WDGRL\":\n",
    "    model = WDGRL(encoder, task,\n",
    "                loss=\"categorical_crossentropy\",\n",
    "                metrics=[\"acc\"],\n",
    "                copy=False,\n",
    "                optimizer=optimizer_task,\n",
    "                optimizer_enc=optimizer_enc,\n",
    "                optimizer_disc=optimizer_disc,\n",
    "                )\n",
    "elif model_name == \"CDAN\":\n",
    "    model = CDAN(encoder, task,\n",
    "                loss=\"categorical_crossentropy\",\n",
    "                metrics=[\"acc\"],\n",
    "                copy=False,\n",
    "                optimizer=optimizer_task,\n",
    "                optimizer_enc=optimizer_enc,\n",
    "                optimizer_disc=optimizer_disc,\n",
    "                )\n",
    "elif model_name == \"CCSA\":\n",
    "    model = CCSA(encoder, task,\n",
    "                loss=\"categorical_crossentropy\",\n",
    "                metrics=[\"acc\"],\n",
    "                copy=False,\n",
    "                optimizer=optimizer_task,\n",
    "                optimizer_enc=optimizer_enc,\n",
    "                optimizer_disc=optimizer_disc,\n",
    "                )\n",
    "elif model_name == \"DANN\":\n",
    "    model = DANN(encoder, task,\n",
    "                loss=\"categorical_crossentropy\",\n",
    "                metrics=[\"acc\"],\n",
    "                copy=False,\n",
    "                optimizer=optimizer_task,\n",
    "                optimizer_enc=optimizer_enc,\n",
    "                optimizer_disc=optimizer_disc,\n",
    "                )\n",
    "elif model_name == \"ADDA\":\n",
    "    model = ADDA(encoder, task,\n",
    "                loss=\"categorical_crossentropy\",\n",
    "                metrics=[\"acc\"],\n",
    "                copy=False,\n",
    "                optimizer=optimizer_task,\n",
    "                optimizer_enc=optimizer_enc,\n",
    "                optimizer_disc=optimizer_disc,\n",
    "                )\n",
    "\n",
    "print(\"MODEL_NAME:\", model.name)\n",
    "model.fit(X=X_source[:-1], y=y_source[:-1], Xt=X_target, yt=y_target, epochs=epochs, batch_size=batch_size, validation_data=(X_val, y_val))\n",
    "\n",
    "acc = model.history.history.get(\"acc\", None) or model.history.history.get(\"disc_acc\", None)\n",
    "val_acc = model.history.history[\"val_acc\"]\n",
    "print(model.score(X_source, y_source))\n",
    "print(model.score(X_val, y_val))\n",
    "print(model.score(X_target, y_target))\n",
    "\n",
    "plt.plot(acc, label=\"Train acc - final value: %.3f\"%acc[-1])\n",
    "plt.plot(val_acc, label=\"Test acc - final value: %.3f\"%val_acc[-1])\n",
    "plt.legend()\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Acc\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5, 0.5]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.history.history.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cdtrans",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15 (default, Nov 24 2022, 15:19:38) \n[GCC 11.2.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "959b82c3a41427bdf7d14d4ba7335271e0c50cfcddd70501934b27dcc36968ad"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
