{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3526e83a-baa5-4278-81ce-e142e0a6d208",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "sys.path.append(Path('./').absolute().parent.__str__())\n",
    "from _datasets import AutoLoad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "5a0264f8-4b67-44e2-8aa9-468ae8b249b5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12, 15)\n",
      "{'a': 'b'}\n"
     ]
    }
   ],
   "source": [
    "class Test():\n",
    "    def __new__(cls, *args, **kwargs):\n",
    "        print(args)\n",
    "        print(kwargs)\n",
    "Test(12, 15, a='b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f0d8ead2-cfa6-4044-8e7a-6b7146bea9cd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import T5TokenizerFast\n",
    "\n",
    "tokenizer = T5TokenizerFast.from_pretrained('google/t5-small-lm-adapt')\n",
    "tokenizer._is_seq2seq = True\n",
    "loader = AutoLoad(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "07c556fd-780d-4aee-a5e9-ad81a474d94b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sentence1', 'sentence2']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loader.glue_helper.get_task_input('stsb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "04feb162-ef3f-42a8-ab00-23d3faea5209",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8165afbb7bcb474e80b9538b0c0c39da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/5749 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95318c2e7b684eabb280fd34d014f1d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e47b3895f4d4f77920c8d82579ec683",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = loader.get_and_map('glue:stsb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "9dcf1e0c-e703-4e30-9dab-bfc54cde7d3f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e703362287be445fa8f3949c592b1c26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/51.8M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d231baabf80401eacf8c400a811c5ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/100730 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c699b3fdf1e468e9ef8a442651d1f7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/10000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91acd57830124beeb29c9869f3b67788",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split:   0%|          | 0/10000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "ds = load_dataset('super_glue', 'record')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c4d652d7-8237-4e5a-85e5-faf39a88eea5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'passage': \"For everyone who has ever thought about shooting their boss - metaphorically, o fcourse - this one is for you. An employee of a Texas armored car company got to do just that this week to 'demonstrate that they take client safety seriously'. And to further that demonstration, the CEO was sitting alone inside the Mercedes-Benz as 12 rounds from an AK-47 rained down upon the SUV. The company, Texas Armoring Corporation, has supplied protected vehicles to the Pope, celebrities like rapper T.I. and actor Steven Segal and oil executives in West Africa, according to My San Antonio. Texas Armoring Corp. & Jason Forston.\\n@highlight\\nTexas Armoring Corporation created a video to show the effectiveness of their armored\\n@highlight\\nCEO R. Trent Kimball sat in the drivers seat of a Mercedes-Benz SUV\\n@highlight\\nTotal of 12 rounds fired at the windscreen\\n@highlight\\nCompany known for working with celebrities, oil barons and even the Pope\",\n",
       " 'query': \"'When it comes to assuring our clients' safety, we take product testing extremely seriously,' @placeholder says in a video taken of the display.\",\n",
       " 'entities': ['Steven Segal',\n",
       "  'Texas Armoring Corp.',\n",
       "  'Trent Kimball',\n",
       "  'Texas Armoring Corporation',\n",
       "  'Texas',\n",
       "  'AK-47',\n",
       "  'Pope',\n",
       "  'Mercedes-Benz',\n",
       "  'San Antonio',\n",
       "  'West Africa',\n",
       "  'rapper T.I.',\n",
       "  'Jason Forston'],\n",
       " 'entity_spans': {'text': ['Texas',\n",
       "   'Mercedes-Benz',\n",
       "   'AK-47',\n",
       "   'Texas Armoring Corporation',\n",
       "   'Pope',\n",
       "   'rapper T.I.',\n",
       "   'Steven Segal',\n",
       "   'West Africa',\n",
       "   'San Antonio',\n",
       "   'Texas Armoring Corp.',\n",
       "   'Jason Forston',\n",
       "   'Texas Armoring Corporation',\n",
       "   'Trent Kimball',\n",
       "   'Mercedes-Benz',\n",
       "   'Pope'],\n",
       "  'start': [128,\n",
       "   313,\n",
       "   348,\n",
       "   393,\n",
       "   460,\n",
       "   483,\n",
       "   505,\n",
       "   540,\n",
       "   569,\n",
       "   582,\n",
       "   605,\n",
       "   631,\n",
       "   735,\n",
       "   778,\n",
       "   929],\n",
       "  'end': [133,\n",
       "   326,\n",
       "   353,\n",
       "   419,\n",
       "   464,\n",
       "   494,\n",
       "   517,\n",
       "   551,\n",
       "   580,\n",
       "   602,\n",
       "   618,\n",
       "   657,\n",
       "   748,\n",
       "   791,\n",
       "   933]},\n",
       " 'answers': ['Trent Kimball'],\n",
       " 'idx': {'passage': 4, 'query': 10}}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds['train'][10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c77ab84e-1cd2-4038-9354-b7f2668bc99d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from evaluate import load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "dc4b8326-43c7-4941-aae5-3cbea1f793cb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'exact_match': 0.0, 'f1_m': 0.0, 'f1_a': 0.0}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric = load('super_glue', 'multirc')\n",
    "metric.compute(\n",
    "    predictions=[{'prediction': 0, 'idx':{'paragraph': 0, 'question': 0, 'answer': 2}}],\n",
    "    references=[1]\n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "13da4dac-ae6f-4a36-a6ed-ebf077eef625",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EvaluationModule(name: \"super_glue\", module_type: \"metric\", features: {'predictions': {'idx': {'answer': Value(dtype='int64', id=None), 'paragraph': Value(dtype='int64', id=None), 'question': Value(dtype='int64', id=None)}, 'prediction': Value(dtype='int64', id=None)}, 'references': Value(dtype='int64', id=None)}, usage: \"\"\"\n",
       "Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\n",
       "Args:\n",
       "    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n",
       "        - for 'record': list of question-answer dictionaries with the following keys:\n",
       "            - 'idx': index of the question as specified by the dataset\n",
       "            - 'prediction_text': the predicted answer text\n",
       "        - for 'multirc': list of question-answer dictionaries with the following keys:\n",
       "            - 'idx': index of the question-answer pair as specified by the dataset\n",
       "            - 'prediction': the predicted answer label\n",
       "        - otherwise: list of predicted labels\n",
       "    references: list of reference labels. Depending on the SuperGLUE subset:\n",
       "        - for 'record': list of question-answers dictionaries with the following keys:\n",
       "            - 'idx': index of the question as specified by the dataset\n",
       "            - 'answers': list of possible answers\n",
       "        - otherwise: list of reference labels\n",
       "Returns: depending on the SuperGLUE subset:\n",
       "    - for 'record':\n",
       "        - 'exact_match': Exact match between answer and gold answer\n",
       "        - 'f1': F1 score\n",
       "    - for 'multirc':\n",
       "        - 'exact_match': Exact match between answer and gold answer\n",
       "        - 'f1_m': Per-question macro-F1 score\n",
       "        - 'f1_a': Average F1 score over all answers\n",
       "    - for 'axb':\n",
       "        'matthews_correlation': Matthew Correlation\n",
       "    - for 'cb':\n",
       "        - 'accuracy': Accuracy\n",
       "        - 'f1': F1 score\n",
       "    - for all others:\n",
       "        - 'accuracy': Accuracy\n",
       "Examples:\n",
       "\n",
       "    >>> super_glue_metric = evaluate.load('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n",
       "    >>> predictions = [0, 1]\n",
       "    >>> references = [0, 1]\n",
       "    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'accuracy': 1.0}\n",
       "\n",
       "    >>> super_glue_metric = evaluate.load('super_glue', 'cb')\n",
       "    >>> predictions = [0, 1]\n",
       "    >>> references = [0, 1]\n",
       "    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'accuracy': 1.0, 'f1': 1.0}\n",
       "\n",
       "    >>> super_glue_metric = evaluate.load('super_glue', 'record')\n",
       "    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n",
       "    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n",
       "    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'exact_match': 1.0, 'f1': 1.0}\n",
       "\n",
       "    >>> super_glue_metric = evaluate.load('super_glue', 'multirc')\n",
       "    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n",
       "    >>> references = [0, 1]\n",
       "    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n",
       "\n",
       "    >>> super_glue_metric = evaluate.load('super_glue', 'axb')\n",
       "    >>> references = [0, 1]\n",
       "    >>> predictions = [0, 1]\n",
       "    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'matthews_correlation': 1.0}\n",
       "\"\"\", stored examples: 0)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "020f35a1-09ec-4ef3-94f4-28144778a3ab",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "from transformers import T5ForConditionalGeneration\n",
    "import torch\n",
    "\n",
    "model = T5ForConditionalGeneration.from_pretrained('google/t5-small-lm-adapt')\n",
    "\n",
    "def mutate_remove_dropout(model):\n",
    "    for module in model.modules():\n",
    "        if isinstance(module, torch.nn.Dropout):\n",
    "            module._backup_p = module.p\n",
    "            module.p = 0\n",
    "            print(module._backup_p)\n",
    "mutate_remove_dropout(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "146e1eb3-f6a6-41d2-ab84-13b62de8983a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deep]",
   "language": "python",
   "name": "conda-env-deep-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}