SuperposPrompt_Thesis/13_additional_table/openai/Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "55d641c5-ae0e-42af-afba-65dab055734e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "OPENAI_TOKEN = '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'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "86ec3895-06b0-4601-a08f-756d286653b3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.schema.messages import HumanMessage, SystemMessage\n",
    "\n",
    "chat = ChatOpenAI(openai_api_key=OPENAI_TOKEN, temperature=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2e75b407-27a6-4651-b240-0b370424d837",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/msadraei/developer/Thesis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "79a19f7f-0c9d-44a5-8089-d89f3e8ac43a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from _datasets.glue_helper import SuperGLUEHelper, GLUEHelper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f57eace5-57d2-4d0c-908d-20c0f5844f8e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "glue_helper = GLUEHelper()\n",
    "superglue_helper = SuperGLUEHelper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "80bc73c9-c8f5-42cb-a024-2b825c0b1bea",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'paragraph': 'While this process moved along, diplomacy continued its rounds. Direct pressure on the Taliban had proved unsuccessful. As one NSC staff note put it, \"Under the Taliban, Afghanistan is not so much a state sponsor of terrorism as it is a state sponsored by terrorists.\" In early 2000, the United States began a high-level effort to persuade Pakistan to use its influence over the Taliban. In January 2000, Assistant Secretary of State Karl Inderfurth and the State Department\\'s counterterrorism coordinator, Michael Sheehan, met with General Musharraf in Islamabad, dangling before him the possibility of a presidential visit in March as a reward for Pakistani cooperation. Such a visit was coveted by Musharraf, partly as a sign of his government\\'s legitimacy. He told the two envoys that he would meet with Mullah Omar and press him on  Bin Laden. They left, however, reporting to Washington that Pakistan was unlikely in fact to do anything,\" given what it sees as the benefits of Taliban control of Afghanistan.\" President Clinton was scheduled to travel to India. The State Department felt that he should not visit India without also visiting Pakistan. The Secret Service and the CIA, however, warned in the strongest terms that visiting Pakistan would risk the President\\'s life. Counterterrorism officials also argued that Pakistan had not done enough to merit a presidential visit. But President Clinton insisted on including Pakistan in the itinerary for his trip to South Asia. His one-day stopover on March 25, 2000, was the first time a U.S. president had been there since 1969. At his meeting with Musharraf and others, President Clinton concentrated on tensions between Pakistan and India and the dangers of nuclear proliferation, but also discussed  Bin Laden. President Clinton told us that when he pulled Musharraf aside for a brief, one-on-one meeting, he pleaded with the general for help regarding  Bin Laden.\" I offered him the moon when I went to see him, in terms of better relations with the United States, if he\\'d help us get  Bin Laden and deal with another issue or two.\" The U.S. effort continued. ',\n",
       " 'question': 'What did the high-level effort to persuade Pakistan include?',\n",
       " 'answer': 'Children, Gerd, or Dorian Popa',\n",
       " 'idx': {'paragraph': 0, 'question': 0, 'answer': 0},\n",
       " 'label': 0}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "superglue_helper.datasets['multirc']['train'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "392f5304-00e8-41ec-aab5-0bd34e6bb3e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "from evaluate import load\n",
    "\n",
    "prompt_template = 'input = {input}\\noutput = {output}'\n",
    "\n",
    "def prepare_wic(input_dict_row):\n",
    "    word = input_dict_row['word']\n",
    "    sent1 = input_dict_row['sentence1']\n",
    "    sent2 = input_dict_row['sentence2']\n",
    "    slice1 = slice(input_dict_row['start1'], input_dict_row['end1'])\n",
    "    slice2 = slice(input_dict_row['start2'], input_dict_row['end2'])\n",
    "\n",
    "    anotate_word = lambda _sent, _slice: _sent[:_slice.start] + \" ** \" + _sent[_slice] + \" ** \" + _sent[_slice.stop:]\n",
    "    input_dict_row['sentence1'] = anotate_word(sent1, slice1)\n",
    "    input_dict_row['sentence2'] = anotate_word(sent2, slice2)\n",
    "\n",
    "    return {\n",
    "        'sentence1': input_dict_row['sentence1'],\n",
    "        'sentence2': input_dict_row['sentence2']\n",
    "    }\n",
    "\n",
    "def make_chatgpt_ready(ds_helper, task_name):\n",
    "    ds = ds_helper.datasets[task_name]\n",
    "    if task_name == 'wic':\n",
    "        ds = {\n",
    "            split: [\n",
    "                {\n",
    "                    **prepare_wic(row),\n",
    "                    'label': row['label'],\n",
    "                    'idx': 0\n",
    "                } for row in ds[split]\n",
    "            ]\n",
    "            for split in ['train', 'validation']\n",
    "        }\n",
    "    if task_name not in ['wic', 'boolq', 'cb', 'copa', 'cola', 'mrpc', 'rte', 'sst2', 'multirc']:\n",
    "        np.random.seed(42)\n",
    "        validation_samples = np.random.choice(range(len(ds['validation'])), replace=False, size=2000).tolist()\n",
    "        ds = {\n",
    "            'train': ds['train'],\n",
    "            'validation': [ds['validation'][idx] for idx in validation_samples]\n",
    "        }\n",
    "    task_out = ds_helper.get_task_output(task_name)\n",
    "    \n",
    "    all_labels = [row['label'] for row in ds['validation']]\n",
    "    if task_name == 'multirc':\n",
    "        all_idx = ds['validation']['idx']\n",
    "        def compute_metric(y_pred):\n",
    "            glue_metric = load(ds_helper.base_name, task_name)\n",
    "            y_pred = [\n",
    "                task_out.str2int(json.loads(item)['label'])\n",
    "                for item in y_pred\n",
    "            ]\n",
    "            assert len(all_idx) == len(y_pred)\n",
    "            y_pred = [\n",
    "                {\n",
    "                    'prediction': y_pred_item,\n",
    "                    'idx': idx\n",
    "                } for (y_pred_item, idx) in zip(y_pred, all_idx)\n",
    "            ]\n",
    "            return glue_metric.compute(predictions=y_pred, references=all_labels)\n",
    "    else:\n",
    "        def compute_metric(y_pred):\n",
    "            glue_metric = load(ds_helper.base_name, task_name)\n",
    "            all_preds = [\n",
    "                task_out.str2int(json.loads(item)['label'])\n",
    "                for item in y_pred\n",
    "            ]\n",
    "            return glue_metric.compute(predictions=all_preds, references=all_labels)\n",
    "        \n",
    "    few_exmples = {}\n",
    "    for row in ds['train']:\n",
    "        if row['label'] not in few_exmples:\n",
    "            label = row.pop('label')\n",
    "            row.pop('idx')\n",
    "            few_exmples[label] = row\n",
    "            \n",
    "    class_names = json.dumps(task_out.names)\n",
    "    pre_prompt_parts = [f'class_names = {class_names}']\n",
    "    for label_id, example in few_exmples.items():\n",
    "        pre_prompt_parts.append(\n",
    "            prompt_template.format(\n",
    "                input = json.dumps(example),\n",
    "                output = json.dumps({'label': task_out.int2str(label_id)})\n",
    "            )\n",
    "        )\n",
    "    \n",
    "    prompt_str = []\n",
    "    for row in ds['validation']:\n",
    "        row.pop('label')\n",
    "        row.pop('idx')\n",
    "        prompt_parts = pre_prompt_parts + [\n",
    "            prompt_template.format(\n",
    "                input = json.dumps(row),\n",
    "                output = ''\n",
    "            )\n",
    "        ]\n",
    "        prompt_str.append('\\n'.join(prompt_parts))\n",
    "    \n",
    "    return prompt_str, compute_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9304b06b-1c8c-4654-b074-c442f3aa3ed4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def make_chatgpt_ready_stsb(ds_helper, task_name):\n",
    "    ds = ds_helper.datasets[task_name]\n",
    "    task_out = ds_helper.get_task_output(task_name)\n",
    "    \n",
    "    all_labels = [row['label'] for row in ds['validation']]\n",
    "    def compute_metric(y_pred):\n",
    "        glue_metric = load(ds_helper.base_name, task_name)\n",
    "        all_preds = [\n",
    "            task_out.str2int(json.loads(item)['label'])\n",
    "            for item in y_pred\n",
    "        ]\n",
    "        return glue_metric.compute(predictions=all_preds, references=all_labels)\n",
    "        \n",
    "    few_exmples = {}\n",
    "    for row in ds['train']:\n",
    "        row['label'] = task_out.int2str(row['label'])\n",
    "        if row['label'] not in few_exmples:\n",
    "            label = row.pop('label')\n",
    "            row.pop('idx')\n",
    "            few_exmples[label] = row\n",
    "            \n",
    "    class_names = list(sorted(few_exmples.keys()))\n",
    "    pre_prompt_parts = [f'class_names = {class_names}']\n",
    "    for label_id, example in few_exmples.items():\n",
    "        pre_prompt_parts.append(\n",
    "            prompt_template.format(\n",
    "                input = json.dumps(example),\n",
    "                output = json.dumps({'label': label_id})\n",
    "            )\n",
    "        )\n",
    "    \n",
    "    prompt_str = []\n",
    "    for row in ds['validation']:\n",
    "        row.pop('label')\n",
    "        row.pop('idx')\n",
    "        prompt_parts = pre_prompt_parts + [\n",
    "            prompt_template.format(\n",
    "                input = json.dumps(row),\n",
    "                output = ''\n",
    "            )\n",
    "        ]\n",
    "        prompt_str.append('\\n'.join(prompt_parts))\n",
    "    \n",
    "    return prompt_str, compute_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "afe4b96f-2948-4544-9397-121a10319bf6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "task_name = 'multirc'\n",
    "prompts, compute_metric = make_chatgpt_ready(superglue_helper, task_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cec4a27-bcfc-4699-9555-9d2cefcdfcaa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "# all_results = []\n",
    "for prompt in tqdm(prompts):\n",
    "    messages = [\n",
    "        SystemMessage(content=\"You are going to be used as a model for natural language understanding task. Read the json input and output carefully and according to the few-shot examples, classify the input. Your output label must be a member of 'class_names'. Your task is according to the paragraph the answer of question is True of False.\"),\n",
    "        HumanMessage(content=prompt)\n",
    "    ]\n",
    "    all_results.append(chat.invoke(messages).content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "57acf17a-8aa1-4f7a-90b3-dd69460d81df",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 504/504 [08:28<00:00,  1.01s/it]\n"
     ]
    }
   ],
   "source": [
    "for prompt in tqdm(prompts[len(all_results):]):\n",
    "    messages = [\n",
    "        SystemMessage(content=\"You are going to be used as a model for natural language understanding task. Read the json input and output carefully and according to the few-shot examples, classify the input. Your output label must be a member of 'class_names'. Your task is according to the paragraph the answer of question is True of False.\"),\n",
    "        HumanMessage(content=prompt)\n",
    "    ]\n",
    "    all_results.append(chat.invoke(messages).content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "8e2ea4da-4710-42fa-befc-0c93fd8e5df0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def conv_res(inp):\n",
    "#     if 'label' in inp:\n",
    "#         return inp\n",
    "#     return json.dumps({'label': inp})\n",
    "\n",
    "# all_results_conv = [conv_res(x) for x in all_results]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "15f18e92-80ca-4b7c-87e6-20d694e8cca1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'exact_match': 0.3410283315844701,\n",
       " 'f1_m': 0.728404774590195,\n",
       " 'f1_a': 0.7791361043194783}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = compute_metric(all_results)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1041840c-4590-4034-8e64-cbdc215a11a8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.555"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.77 + 0.34) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6171134d-45ba-4bc8-991c-8fbd1cb7d370",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(f'./{task_name}.json', 'w') as f:\n",
    "    json.dump(result, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "2fca5a91-dbba-4768-9b9f-82f56619f2fb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'class_names = [\"False\", \"True\"]\\ninput = {\"sentence1\": \"Do you want to come over to my  ** place **  later?\", \"sentence2\": \"A political system with no  ** place **  for the less prominent groups.\"}\\noutput = {\"label\": \"False\"}\\ninput = {\"sentence1\": \"The general ordered the colonel to  ** hold **  his position at all costs.\", \"sentence2\": \" ** Hold **  the taxi.\"}\\noutput = {\"label\": \"True\"}\\ninput = {\"sentence1\": \"An emerging professional  ** class ** .\", \"sentence2\": \"Apologizing for losing your temper, even though you were badly provoked, showed real  ** class ** .\"}\\noutput = '"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "229572a2-20ac-43d6-b370-7812deef23cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:openai]",
   "language": "python",
   "name": "conda-env-openai-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}