Persian-G2P-Tools-Benchmark/benchmarking-scripts/Benchmark_de_mh_persian_phonemizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WEY5MiKLzurH"
      },
      "source": [
        "# Setup Environment"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install hazm==0.10.0"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "QtQ-huXYtepn",
        "outputId": "7755d71f-82c4-4a8f-dd14-919767ccd82a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting hazm==0.10.0\n",
            "  Downloading hazm-0.10.0-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting fasttext-wheel<0.10.0,>=0.9.2 (from hazm==0.10.0)\n",
            "  Downloading fasttext_wheel-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB)\n",
            "Collecting flashtext<3.0,>=2.7 (from hazm==0.10.0)\n",
            "  Downloading flashtext-2.7.tar.gz (14 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gensim<5.0.0,>=4.3.1 (from hazm==0.10.0)\n",
            "  Downloading gensim-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (8.1 kB)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.11/dist-packages (from hazm==0.10.0) (3.9.1)\n",
            "Collecting numpy==1.24.3 (from hazm==0.10.0)\n",
            "  Downloading numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB)\n",
            "Collecting python-crfsuite<0.10.0,>=0.9.9 (from hazm==0.10.0)\n",
            "  Downloading python_crfsuite-0.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.3 kB)\n",
            "Requirement already satisfied: scikit-learn<2.0.0,>=1.2.2 in /usr/local/lib/python3.11/dist-packages (from hazm==0.10.0) (1.6.1)\n",
            "Collecting pybind11>=2.2 (from fasttext-wheel<0.10.0,>=0.9.2->hazm==0.10.0)\n",
            "  Downloading pybind11-2.13.6-py3-none-any.whl.metadata (9.5 kB)\n",
            "Requirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from fasttext-wheel<0.10.0,>=0.9.2->hazm==0.10.0) (75.2.0)\n",
            "Collecting scipy<1.14.0,>=1.7.0 (from gensim<5.0.0,>=4.3.1->hazm==0.10.0)\n",
            "  Downloading scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.11/dist-packages (from gensim<5.0.0,>=4.3.1->hazm==0.10.0) (7.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm==0.10.0) (8.1.8)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm==0.10.0) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm==0.10.0) (2024.11.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm==0.10.0) (4.67.1)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<2.0.0,>=1.2.2->hazm==0.10.0) (3.6.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.11/dist-packages (from smart-open>=1.8.1->gensim<5.0.0,>=4.3.1->hazm==0.10.0) (1.17.2)\n",
            "Downloading hazm-0.10.0-py3-none-any.whl (892 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m892.6/892.6 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m72.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fasttext_wheel-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m77.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gensim-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (26.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.7/26.7 MB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_crfsuite-0.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m37.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pybind11-2.13.6-py3-none-any.whl (243 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m243.3/243.3 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.6/38.6 MB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: flashtext\n",
            "  Building wheel for flashtext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for flashtext: filename=flashtext-2.7-py2.py3-none-any.whl size=9300 sha256=ced3adb209606d785c23d09dcadd043fd23f7c63a88a04dc1a3922017818b447\n",
            "  Stored in directory: /root/.cache/pip/wheels/49/20/47/f03dfa8a7239c54cbc44ff7389eefbf888d2c1873edaaec888\n",
            "Successfully built flashtext\n",
            "Installing collected packages: flashtext, python-crfsuite, pybind11, numpy, scipy, fasttext-wheel, gensim, hazm\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 2.0.2\n",
            "    Uninstalling numpy-2.0.2:\n",
            "      Successfully uninstalled numpy-2.0.2\n",
            "  Attempting uninstall: scipy\n",
            "    Found existing installation: scipy 1.15.2\n",
            "    Uninstalling scipy-1.15.2:\n",
            "      Successfully uninstalled scipy-1.15.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "blosc2 3.3.2 requires numpy>=1.26, but you have numpy 1.24.3 which is incompatible.\n",
            "thinc 8.3.6 requires numpy<3.0.0,>=2.0.0, but you have numpy 1.24.3 which is incompatible.\n",
            "treescope 0.1.9 requires numpy>=1.25.2, but you have numpy 1.24.3 which is incompatible.\n",
            "pymc 5.22.0 requires numpy>=1.25.0, but you have numpy 1.24.3 which is incompatible.\n",
            "albumentations 2.0.6 requires numpy>=1.24.4, but you have numpy 1.24.3 which is incompatible.\n",
            "albucore 0.0.24 requires numpy>=1.24.4, but you have numpy 1.24.3 which is incompatible.\n",
            "tensorflow 2.18.0 requires numpy<2.1.0,>=1.26.0, but you have numpy 1.24.3 which is incompatible.\n",
            "jax 0.5.2 requires numpy>=1.25, but you have numpy 1.24.3 which is incompatible.\n",
            "jaxlib 0.5.1 requires numpy>=1.25, but you have numpy 1.24.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed fasttext-wheel-0.9.2 flashtext-2.7 gensim-4.3.3 hazm-0.10.0 numpy-1.24.3 pybind11-2.13.6 python-crfsuite-0.9.11 scipy-1.13.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "numpy"
                ]
              },
              "id": "954f89a9223948ee9ece2a3af9016355"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install persian_phonemizer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JJ9VISNqZS6m",
        "outputId": "56e5c328-ecda-429f-e8b7-d9044dae080a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting persian_phonemizer\n",
            "  Downloading persian_phonemizer-0.4.0-py3-none-any.whl.metadata (3.1 kB)\n",
            "Requirement already satisfied: spacy>=3.3 in /usr/local/lib/python3.11/dist-packages (from persian_phonemizer) (3.8.5)\n",
            "Collecting g2p-fa (from persian_phonemizer)\n",
            "  Downloading g2p_fa-1.1.0-py3-none-any.whl.metadata (2.7 kB)\n",
            "Requirement already satisfied: hazm in /usr/local/lib/python3.11/dist-packages (from persian_phonemizer) (0.10.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (1.0.12)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (2.0.11)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.4.0,>=8.3.4 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (8.3.6)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (1.1.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (2.5.1)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.5.0,>=0.1.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (0.4.1)\n",
            "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (0.15.3)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (4.67.1)\n",
            "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (1.24.3)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (2.32.3)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (2.11.4)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (3.1.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (75.2.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (24.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.11/dist-packages (from spacy>=3.3->persian_phonemizer) (3.5.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.11/dist-packages (from g2p-fa->persian_phonemizer) (2.6.0+cu124)\n",
            "Requirement already satisfied: fasttext-wheel<0.10.0,>=0.9.2 in /usr/local/lib/python3.11/dist-packages (from hazm->persian_phonemizer) (0.9.2)\n",
            "Requirement already satisfied: flashtext<3.0,>=2.7 in /usr/local/lib/python3.11/dist-packages (from hazm->persian_phonemizer) (2.7)\n",
            "Requirement already satisfied: gensim<5.0.0,>=4.3.1 in /usr/local/lib/python3.11/dist-packages (from hazm->persian_phonemizer) (4.3.3)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.11/dist-packages (from hazm->persian_phonemizer) (3.9.1)\n",
            "Requirement already satisfied: python-crfsuite<0.10.0,>=0.9.9 in /usr/local/lib/python3.11/dist-packages (from hazm->persian_phonemizer) (0.9.11)\n",
            "Requirement already satisfied: scikit-learn<2.0.0,>=1.2.2 in /usr/local/lib/python3.11/dist-packages (from hazm->persian_phonemizer) (1.6.1)\n",
            "Requirement already satisfied: pybind11>=2.2 in /usr/local/lib/python3.11/dist-packages (from fasttext-wheel<0.10.0,>=0.9.2->hazm->persian_phonemizer) (2.13.6)\n",
            "Requirement already satisfied: scipy<1.14.0,>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from gensim<5.0.0,>=4.3.1->hazm->persian_phonemizer) (1.13.1)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.11/dist-packages (from gensim<5.0.0,>=4.3.1->hazm->persian_phonemizer) (7.1.0)\n",
            "Requirement already satisfied: language-data>=1.2 in /usr/local/lib/python3.11/dist-packages (from langcodes<4.0.0,>=3.2.0->spacy>=3.3->persian_phonemizer) (1.3.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm->persian_phonemizer) (8.1.8)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm->persian_phonemizer) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm->persian_phonemizer) (2024.11.6)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy>=3.3->persian_phonemizer) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.33.2 in /usr/local/lib/python3.11/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy>=3.3->persian_phonemizer) (2.33.2)\n",
            "Requirement already satisfied: typing-extensions>=4.12.2 in /usr/local/lib/python3.11/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy>=3.3->persian_phonemizer) (4.13.2)\n",
            "Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy>=3.3->persian_phonemizer) (0.4.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=3.3->persian_phonemizer) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=3.3->persian_phonemizer) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=3.3->persian_phonemizer) (2.4.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=3.3->persian_phonemizer) (2025.4.26)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<2.0.0,>=1.2.2->hazm->persian_phonemizer) (3.6.0)\n",
            "Requirement already satisfied: blis<1.4.0,>=1.3.0 in /usr/local/lib/python3.11/dist-packages (from thinc<8.4.0,>=8.3.4->spacy>=3.3->persian_phonemizer) (1.3.0)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.11/dist-packages (from thinc<8.4.0,>=8.3.4->spacy>=3.3->persian_phonemizer) (0.1.5)\n",
            "INFO: pip is looking at multiple versions of thinc to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting thinc<8.4.0,>=8.3.4 (from spacy>=3.3->persian_phonemizer)\n",
            "  Downloading thinc-8.3.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (15 kB)\n",
            "Collecting blis<1.3.0,>=1.2.0 (from thinc<8.4.0,>=8.3.4->spacy>=3.3->persian_phonemizer)\n",
            "  Downloading blis-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.4 kB)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.11/dist-packages (from typer<1.0.0,>=0.3.0->spacy>=3.3->persian_phonemizer) (1.5.4)\n",
            "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.11/dist-packages (from typer<1.0.0,>=0.3.0->spacy>=3.3->persian_phonemizer) (13.9.4)\n",
            "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from weasel<0.5.0,>=0.1.0->spacy>=3.3->persian_phonemizer) (0.21.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->spacy>=3.3->persian_phonemizer) (3.0.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (3.18.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (3.4.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (2025.3.2)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.4.127 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.4.127 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.4.127 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cudnn-cu12==9.1.0.70 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.4.5.8 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cufft-cu12==11.2.1.3 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-curand-cu12==10.3.5.147 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cusolver-cu12==11.6.1.9 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cusparse-cu12==12.3.1.170 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n",
            "Requirement already satisfied: nvidia-cusparselt-cu12==0.6.2 in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (0.6.2)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (2.21.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (12.4.127)\n",
            "Collecting nvidia-nvjitlink-cu12==12.4.127 (from torch->g2p-fa->persian_phonemizer)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: triton==3.2.0 in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (3.2.0)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.11/dist-packages (from torch->g2p-fa->persian_phonemizer) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy==1.13.1->torch->g2p-fa->persian_phonemizer) (1.3.0)\n",
            "Requirement already satisfied: marisa-trie>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy>=3.3->persian_phonemizer) (1.2.1)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.11/dist-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy>=3.3->persian_phonemizer) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy>=3.3->persian_phonemizer) (2.19.1)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.11/dist-packages (from smart-open>=1.8.1->gensim<5.0.0,>=4.3.1->hazm->persian_phonemizer) (1.17.2)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.11/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy>=3.3->persian_phonemizer) (0.1.2)\n",
            "Downloading persian_phonemizer-0.4.0-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m62.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading g2p_fa-1.1.0-py3-none-any.whl (6.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m104.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading thinc-8.3.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.9/3.9 MB\u001b[0m \u001b[31m98.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl (363.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m363.4/363.4 MB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (13.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.8/13.8 MB\u001b[0m \u001b[31m104.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (24.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.6/24.6 MB\u001b[0m \u001b[31m80.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (883 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m883.7/883.7 kB\u001b[0m \u001b[31m50.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl (664.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m664.8/664.8 MB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl (211.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.5/211.5 MB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl (56.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.3/56.3 MB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl (127.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m127.9/127.9 MB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl (207.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.5/207.5 MB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m90.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading blis-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.7/11.7 MB\u001b[0m \u001b[31m112.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: nvidia-nvjitlink-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, blis, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, thinc, g2p-fa, persian_phonemizer\n",
            "  Attempting uninstall: nvidia-nvjitlink-cu12\n",
            "    Found existing installation: nvidia-nvjitlink-cu12 12.5.82\n",
            "    Uninstalling nvidia-nvjitlink-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-nvjitlink-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-curand-cu12\n",
            "    Found existing installation: nvidia-curand-cu12 10.3.6.82\n",
            "    Uninstalling nvidia-curand-cu12-10.3.6.82:\n",
            "      Successfully uninstalled nvidia-curand-cu12-10.3.6.82\n",
            "  Attempting uninstall: nvidia-cufft-cu12\n",
            "    Found existing installation: nvidia-cufft-cu12 11.2.3.61\n",
            "    Uninstalling nvidia-cufft-cu12-11.2.3.61:\n",
            "      Successfully uninstalled nvidia-cufft-cu12-11.2.3.61\n",
            "  Attempting uninstall: nvidia-cuda-runtime-cu12\n",
            "    Found existing installation: nvidia-cuda-runtime-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-runtime-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-runtime-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cuda-nvrtc-cu12\n",
            "    Found existing installation: nvidia-cuda-nvrtc-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-nvrtc-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-nvrtc-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cuda-cupti-cu12\n",
            "    Found existing installation: nvidia-cuda-cupti-cu12 12.5.82\n",
            "    Uninstalling nvidia-cuda-cupti-cu12-12.5.82:\n",
            "      Successfully uninstalled nvidia-cuda-cupti-cu12-12.5.82\n",
            "  Attempting uninstall: nvidia-cublas-cu12\n",
            "    Found existing installation: nvidia-cublas-cu12 12.5.3.2\n",
            "    Uninstalling nvidia-cublas-cu12-12.5.3.2:\n",
            "      Successfully uninstalled nvidia-cublas-cu12-12.5.3.2\n",
            "  Attempting uninstall: blis\n",
            "    Found existing installation: blis 1.3.0\n",
            "    Uninstalling blis-1.3.0:\n",
            "      Successfully uninstalled blis-1.3.0\n",
            "  Attempting uninstall: nvidia-cusparse-cu12\n",
            "    Found existing installation: nvidia-cusparse-cu12 12.5.1.3\n",
            "    Uninstalling nvidia-cusparse-cu12-12.5.1.3:\n",
            "      Successfully uninstalled nvidia-cusparse-cu12-12.5.1.3\n",
            "  Attempting uninstall: nvidia-cudnn-cu12\n",
            "    Found existing installation: nvidia-cudnn-cu12 9.3.0.75\n",
            "    Uninstalling nvidia-cudnn-cu12-9.3.0.75:\n",
            "      Successfully uninstalled nvidia-cudnn-cu12-9.3.0.75\n",
            "  Attempting uninstall: nvidia-cusolver-cu12\n",
            "    Found existing installation: nvidia-cusolver-cu12 11.6.3.83\n",
            "    Uninstalling nvidia-cusolver-cu12-11.6.3.83:\n",
            "      Successfully uninstalled nvidia-cusolver-cu12-11.6.3.83\n",
            "  Attempting uninstall: thinc\n",
            "    Found existing installation: thinc 8.3.6\n",
            "    Uninstalling thinc-8.3.6:\n",
            "      Successfully uninstalled thinc-8.3.6\n",
            "Successfully installed blis-1.2.1 g2p-fa-1.1.0 nvidia-cublas-cu12-12.4.5.8 nvidia-cuda-cupti-cu12-12.4.127 nvidia-cuda-nvrtc-cu12-12.4.127 nvidia-cuda-runtime-cu12-12.4.127 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.2.1.3 nvidia-curand-cu12-10.3.5.147 nvidia-cusolver-cu12-11.6.1.9 nvidia-cusparse-cu12-12.3.1.170 nvidia-nvjitlink-cu12-12.4.127 persian_phonemizer-0.4.0 thinc-8.3.4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install jiwer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5OD5ZtfTck77",
        "outputId": "775bc630-e1d3-49d1-9e0f-62f96dc65ee5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting jiwer\n",
            "  Downloading jiwer-3.1.0-py3-none-any.whl.metadata (2.6 kB)\n",
            "Requirement already satisfied: click>=8.1.8 in /usr/local/lib/python3.11/dist-packages (from jiwer) (8.1.8)\n",
            "Collecting rapidfuzz>=3.9.7 (from jiwer)\n",
            "  Downloading rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Downloading jiwer-3.1.0-py3-none-any.whl (22 kB)\n",
            "Downloading rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m35.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: rapidfuzz, jiwer\n",
            "Successfully installed jiwer-3.1.0 rapidfuzz-3.13.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "from tqdm import tqdm\n",
        "import csv\n",
        "import pandas as pd\n",
        "import json\n",
        "import itertools\n",
        "from jiwer import cer"
      ],
      "metadata": {
        "id": "DLEJ-KSecpyV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from persian_phonemizer import Phonemizer\n",
        "phonemizer = Phonemizer()\n",
        "phonemizer.phonemize('دلم میخواست برم ')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "my5HlSvaaz6h",
        "outputId": "6f4c048f-412b-4940-97a7-6a6eb284881a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'dele mæʒ miːxævɒːstʰ bæɾm'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# mapping"
      ],
      "metadata": {
        "id": "VtxEYym69RUH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output_to_phonetics_map = {\n",
        "    'м': 'm',\n",
        "    'ʷ':' v',\n",
        "    'c': 'k',\n",
        "    'ĉ': 'C',\n",
        "    'č': 'C',\n",
        "    '̕': \"?\",\n",
        "    \"'\": '?',\n",
        "    'ʔ': \"?\",\n",
        "    'ꞌ': \"?\",\n",
        "    '̛':  \"?\",\n",
        "    '’': \"?\",\n",
        "    'ʼ': \"?\",\n",
        "    \"'\": '?',\n",
        "    'â': 'A',\n",
        "    'â': 'A',\n",
        "    'ȃ': 'A',\n",
        "    'ž': 'Z',\n",
        "    'š': 'S',\n",
        "    'W': 'v',\n",
        "    'β': 'f',\n",
        "    'е': 'e',\n",
        "    '`': \"?\",\n",
        "    'ɑ': 'A',\n",
        "    'ɑ': 'A',\n",
        "    'ʃ': 'S',\n",
        "    'ð': 'z',\n",
        "    'ɾ': 'r',\n",
        "    'æ': 'a',\n",
        "    'ɪ': 'e',\n",
        "    'χ': 'x',\n",
        "    'ɣ': 'q',\n",
        "    'ʒ': 'Z',\n",
        "    ':': '',\n",
        "    'ː': '',\n",
        "    'ā': 'A',\n",
        "    'ː': '',\n",
        "    'ä': 'A',\n",
        "    'á': 'A',\n",
        "    'š': 'S',\n",
        "    'ū': 'u',\n",
        "    'û': 'u',\n",
        "    'ś': 's',\n",
        "    'ī': 'i',\n",
        "    'í': 'i',\n",
        "    'î': 'i',\n",
        "    'é': 'e',\n",
        "    'ḥ': 'h',\n",
        "    'ɒ': 'A',\n",
        "    'ʰ': '',\n",
        "    'ə': 'e',\n",
        "    'R': 'r',\n",
        "    'W': 'v',\n",
        "    'Q': 'q',\n",
        "    'T': 't',\n",
        "    'Y': 'y',\n",
        "    'P': 'p',\n",
        "    'D': 'd',\n",
        "    'F': 'f',\n",
        "    'H': 'h',\n",
        "    'J': 'j',\n",
        "    'L': 'l',\n",
        "    'X': 'x',\n",
        "    'V': 'v',\n",
        "    'B': 'b',\n",
        "    'N': 'n',\n",
        "    'M': 'm',\n",
        "    'K': 'k',\n",
        "    'G': 'g',\n",
        "    'U': 'u',\n",
        "    'O': 'o',\n",
        "    'I': 'i',\n",
        "    'E': 'e',\n",
        "    'ŋ': 'ng',\n",
        "    '.': '',\n",
        "    'ɛ': 'e',\n",
        "    'ʊ': 'u',\n",
        "    \"ˈ\": '?',\n",
        "    'ù': 'u',\n",
        "    'θ': 's',\n",
        "    '̪': '',\n",
        "    'ũ': 'u',\n",
        "    '_': '',\n",
        "    'ç': 'C',\n",
        "    'ĝ': 'q',\n",
        "    'ɢ': 'q',\n",
        "    'ː': '',\n",
        "    'í': 'i',\n",
        "    'ŝ': 'S',\n",
        "    '!': '',\n",
        "    'ǧ': 'q',\n",
        "    'ʻ': '?',\n",
        "    'è': 'e',\n",
        "    '�': '',\n",
        "    'ú': 'u',\n",
        "    'ô': 'o',\n",
        "    'ē': 'e',\n",
        "    'à': 'A',\n",
        "    'ă': 'A',\n",
        "    'ǐ': 'i',\n",
        "    'ü': 'u',\n",
        "    '\\u200e': '',\n",
        "    'ğ': 'q',\n",
        "    'ṣ': 'S',\n",
        "    'â': 'A',\n",
        "    'â': 'A',\n",
        "    'ȃ': 'A',\n",
        "    'ž': 'Z',\n",
        "    'š': 'S',\n",
        "    'ā': 'A',\n",
        "    'ː': '',\n",
        "    'ä': 'A',\n",
        "    'á': 'A',\n",
        "    'š': 'S',\n",
        "    'ū': 'u',\n",
        "    'û': 'u',\n",
        "    'ś': 'S',\n",
        "    'ī': 'i',\n",
        "    'í': 'i',\n",
        "    'î': 'i',\n",
        "    'é': 'e',\n",
        "}\n",
        "\n",
        "consonants_regex = '(?=' + '|'.join(['q', 'r', 't', 'y', 'p', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'z', 'x', 'c', 'v', 'b', 'n', 'm', 'Q', 'R', 'T', 'Y', 'P', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L', 'Z', 'X', 'C', 'V', 'B', 'N', 'M' ]) + ')'\n",
        "vowels_regex = '(?=' + '|'.join(['a', 'A', 'e', 'i', 'u', 'o']) + ')'\n",
        "\n",
        "\n",
        "def replace_phonetic_characters(input_string, char_map=output_to_phonetics_map, from_phonetics=False):\n",
        "    substituted = re.sub(r'tʃʰ', 'C', input_string)\n",
        "    substituted = re.sub(r't͡ʃ', 'C', input_string)\n",
        "    substituted = re.sub(r'tʃ', 'C', substituted)\n",
        "    substituted = re.sub(r't͡S', 'C', substituted)\n",
        "    substituted = re.sub(r'ow', 'o', substituted)\n",
        "    substituted = re.sub('d͡ʒ', 'j', substituted)\n",
        "    substituted = re.sub('dʒ', 'j', substituted)\n",
        "\n",
        "    # Create a translation table using str.maketrans\n",
        "    translation_table = str.maketrans(char_map)\n",
        "\n",
        "    # Use str.translate to replace characters based on the translation table\n",
        "    translated = substituted.translate(translation_table)\n",
        "\n",
        "    return translated"
      ],
      "metadata": {
        "id": "TKx8oA1n7rKh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XjAPkfq7SF87"
      },
      "source": [
        "# Get Evaluation Data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://huggingface.co/datasets/MahtaFetrat/SentenceBench/raw/main/SentenceBench.csv"
      ],
      "metadata": {
        "id": "qwCG0jX-88nQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "63d62243-fa2f-4241-bc6c-a9d5b24f8980"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2025-05-10 10:07:38--  https://huggingface.co/datasets/MahtaFetrat/SentenceBench/raw/main/SentenceBench.csv\n",
            "Resolving huggingface.co (huggingface.co)... 18.160.143.76, 18.160.143.99, 18.160.143.75, ...\n",
            "Connecting to huggingface.co (huggingface.co)|18.160.143.76|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 56026 (55K) [text/plain]\n",
            "Saving to: ‘SentenceBench.csv’\n",
            "\n",
            "\rSentenceBench.csv     0%[                    ]       0  --.-KB/s               \rSentenceBench.csv   100%[===================>]  54.71K  --.-KB/s    in 0.01s   \n",
            "\n",
            "2025-05-10 10:07:38 (4.42 MB/s) - ‘SentenceBench.csv’ saved [56026/56026]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence_bench = pd.read_csv('SentenceBench.csv')"
      ],
      "metadata": {
        "id": "hJO-UAPDQvcb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence_bench.head(3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "qlYbrnUa9LAN",
        "outputId": "584e795c-1801-4594-be3f-e4426a16925f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     dataset                              grapheme  \\\n",
              "0  homograph                  من قدر تو را می‌دانم   \n",
              "1  homograph  از قضای الهی به قدر الهی پناه می‌برم   \n",
              "2  homograph                به دست و صورتم کرم زدم   \n",
              "\n",
              "                                             phoneme homograph word  \\\n",
              "0                          man qadr-e to rA mi-dAnam            قدر   \n",
              "1  ?az qazAy ?elAhi be qadar-e ?elAhi panAh mi-baram            قدر   \n",
              "2                      be dast-o suratam kerem zadam            کرم   \n",
              "\n",
              "  pronunciation  \n",
              "0          qadr  \n",
              "1         qadar  \n",
              "2         kerem  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-75a41682-7a05-40b4-8ffb-951442296f9c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>dataset</th>\n",
              "      <th>grapheme</th>\n",
              "      <th>phoneme</th>\n",
              "      <th>homograph word</th>\n",
              "      <th>pronunciation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>homograph</td>\n",
              "      <td>من قدر تو را می‌دانم</td>\n",
              "      <td>man qadr-e to rA mi-dAnam</td>\n",
              "      <td>قدر</td>\n",
              "      <td>qadr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>homograph</td>\n",
              "      <td>از قضای الهی به قدر الهی پناه می‌برم</td>\n",
              "      <td>?az qazAy ?elAhi be qadar-e ?elAhi panAh mi-baram</td>\n",
              "      <td>قدر</td>\n",
              "      <td>qadar</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>homograph</td>\n",
              "      <td>به دست و صورتم کرم زدم</td>\n",
              "      <td>be dast-o suratam kerem zadam</td>\n",
              "      <td>کرم</td>\n",
              "      <td>kerem</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-75a41682-7a05-40b4-8ffb-951442296f9c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-75a41682-7a05-40b4-8ffb-951442296f9c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-75a41682-7a05-40b4-8ffb-951442296f9c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "    <div id=\"df-f541734f-766a-44ba-88c6-1b4f4c4c57b7\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f541734f-766a-44ba-88c6-1b4f4c4c57b7')\"\n",
              "                title=\"Suggest charts\"\n",
              "                style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "      <script>\n",
              "        async function quickchart(key) {\n",
              "          const quickchartButtonEl =\n",
              "            document.querySelector('#' + key + ' button');\n",
              "          quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "          quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "          try {\n",
              "            const charts = await google.colab.kernel.invokeFunction(\n",
              "                'suggestCharts', [key], {});\n",
              "          } catch (error) {\n",
              "            console.error('Error during call to suggestCharts:', error);\n",
              "          }\n",
              "          quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "          quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "        }\n",
              "        (() => {\n",
              "          let quickchartButtonEl =\n",
              "            document.querySelector('#df-f541734f-766a-44ba-88c6-1b4f4c4c57b7 button');\n",
              "          quickchartButtonEl.style.display =\n",
              "            google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "        })();\n",
              "      </script>\n",
              "    </div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "sentence_bench",
              "summary": "{\n  \"name\": \"sentence_bench\",\n  \"rows\": 400,\n  \"fields\": [\n    {\n      \"column\": \"dataset\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"homograph\",\n          \"mana-tts\",\n          \"commonvoice\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"grapheme\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 400,\n        \"samples\": [\n          \"\\u0622\\u06cc\\u0627 \\u0628\\u0627\\u06cc\\u062f \\u062d\\u0642\\u06cc\\u0642\\u062a \\u0631\\u0627 \\u0628\\u0647 \\u0622\\u0646\\u200c\\u0647\\u0627 \\u0628\\u06af\\u0648\\u06cc\\u06cc\\u0645\\u061f\",\n          \"\\u06a9\\u0647 \\u067e\\u06cc\\u0634 \\u0627\\u0632 \\u0627\\u0646\\u0642\\u0644\\u0627\\u0628 \\u0628\\u0647 \\u062e\\u0648\\u0627\\u0628\\u06af\\u0627\\u0647 \\u062f\\u062e\\u062a\\u0631\\u0627\\u0646 \\u0648 \\u0632\\u0646\\u0627\\u0646 \\u0646\\u0627\\u0628\\u06cc\\u0646\\u0627 \\u0627\\u062e\\u062a\\u0635\\u0627\\u0635\\u200c\\u06cc\\u0627\\u0641\\u062a\\u0647 \\u0628\\u0648\\u062f. \\u0627\\u063a\\u0644\\u0628 \\u0632\\u0646\\u0627\\u0646\\u06cc \\u06a9\\u0647 \\u062f\\u0631 \\u0627\\u06cc\\u0646 \\u062e\\u0648\\u0627\\u0628\\u06af\\u0627\\u0647 \\u0632\\u0646\\u062f\\u06af\\u06cc \\u0645\\u06cc\\u200c\\u06a9\\u0631\\u062f\\u0646\\u062f\\u060c \",\n          \"\\u062f\\u0648\\u062f \\u0648 \\u0645\\u0647 \\u063a\\u0644\\u06cc\\u0638\\u06cc \\u062f\\u0631 \\u0645\\u062d\\u06cc\\u0637 \\u067e\\u06cc\\u0686\\u06cc\\u062f\\u0647 \\u0628\\u0648\\u062f\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"phoneme\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 400,\n        \"samples\": [\n          \"?AyA bAyad haqiqat rA be ?AnhA beguyim\\u061f\",\n          \"ke piS ?az ?enqelAb be xAbgAh-e doxtarAn va zanAn-e nAbinA ?extesAsyAfte bud ?aqlab-e zanAni ke dar ?in xAbgAh zendegi mikardand\",\n          \"dud-o meh-e qalizi dar mohit piCide bud\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"homograph word\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 101,\n        \"samples\": [\n          \"\\u06af\\u0631\\u06cc\\u0645\",\n          \"\\u0633\\u0628\\u06a9\\u06cc\",\n          \"\\u06a9\\u0645\\u06cc\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pronunciation\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 210,\n        \"samples\": [\n          \"darham\",\n          \"Sum\",\n          \"moSk\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wDV7ysXf2b_H"
      },
      "source": [
        "### Get ManaTTS"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TcL5ZLvSSnVB",
        "outputId": "57eed88d-7ae2-4c1d-fd40-4f8f09e8dd03"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('در این نوشته بنا داریم با یک ابزار ساده و مکانیکی افزایش بینایی برای افراد کم\\u200cبینا ',\n",
              "  'dar ?in neveSte banA dArim bA yek ?abzAr-e sAde va mekAniki-ye ?afzAyeS-e binAyi barAye ?afrAd-e kam\\u200cbinA '),\n",
              " ('به نام بی\\u200cوپتیک یا عدسی دورنما آشنا شویم. ',\n",
              "  'be nAm-e biyoptik yA ?adasi-ye durnamA ?ASnA Savim'),\n",
              " ('دراین\\u200cصورت، انجام خودارزیابی و ارائه بازخورد بر عهده خودتان است. ',\n",
              "  'dar ?in surat ?anjAm-e xod?arzyAbi va ?erA?e-ye bAzxord bar ?ohde-ye xodetAn ?ast ')]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "filtered_rows = sentence_bench[sentence_bench['dataset'] == 'mana-tts'][['grapheme', 'phoneme']]\n",
        "\n",
        "# Convert to a list of tuples\n",
        "mana_evaluation_data = list(filtered_rows.itertuples(index=False, name=None))\n",
        "\n",
        "mana_evaluation_data[:3]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jjacw9Mp2eoX"
      },
      "source": [
        "### Get CommonVoice"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-yQnqCGw26sk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60f4e58c-8529-4d62-88ca-795413c40385"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('در اکثر شهرها، مرکزی برای خرید دوچرخه وجود دارد.',\n",
              "  'dar ?aksar-e Sahr-hA, markazi barAye xarid-e  doCarxe vojud dArad.'),\n",
              " ('پس از مدرسه کودکان به سوی خانه جست و خیز کردند.',\n",
              "  'pas ?az madrese kudakAn be suye xAne jast-o-xiz kardand.'),\n",
              " ('شما نگران زن و بچه این نباش.', 'SomA negarAn-e zan-o-baCCe-ye ?in nabAS.')]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "filtered_rows = sentence_bench[sentence_bench['dataset'] == 'commonvoice'][['grapheme', 'phoneme']]\n",
        "\n",
        "# Convert to a list of tuples\n",
        "commonvoice_evaluation_data = list(filtered_rows.itertuples(index=False, name=None))\n",
        "\n",
        "commonvoice_evaluation_data[:3]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ciSPyhRc3Rvo"
      },
      "source": [
        "### Get Homograph"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XlFc5JbN3Rvz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "19aeaf85-6001-4d7f-e46f-e65b05b0a861"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('من قدر تو را می\\u200cدانم', 'man qadr-e to rA mi-dAnam', 'قدر', 'qadr'),\n",
              " ('از قضای الهی به قدر الهی پناه می\\u200cبرم',\n",
              "  '?az qazAy ?elAhi be qadar-e ?elAhi panAh mi-baram',\n",
              "  'قدر',\n",
              "  'qadar'),\n",
              " ('به دست و صورتم کرم زدم', 'be dast-o suratam kerem zadam', 'کرم', 'kerem')]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "filtered_rows = sentence_bench[sentence_bench['dataset'] == 'homograph'][['grapheme', 'phoneme', 'homograph word',\t'pronunciation']]\n",
        "\n",
        "# Convert to a list of tuples\n",
        "homograph_evaluation_data = list(filtered_rows.itertuples(index=False, name=None))\n",
        "\n",
        "homograph_evaluation_data[:3]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R6PE5ds45TPr"
      },
      "source": [
        "# Evaluate Method Outputs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CLKaERek4u_D"
      },
      "source": [
        "## PER Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nBee9xG54u_E"
      },
      "outputs": [],
      "source": [
        "def remove_non_word_chars(text):\n",
        "    pattern = r'[^\\w\\s\\?]'\n",
        "    cleaned_text = re.sub(pattern, ' ', text)\n",
        "    return cleaned_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W8PoNV9V4u_E"
      },
      "outputs": [],
      "source": [
        "def remove_white_spaces(text):\n",
        "    cleaned_text = re.sub(r'\\s+', ' ', text)\n",
        "    return cleaned_text.strip()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YD0cvnn74u_E"
      },
      "outputs": [],
      "source": [
        "def get_word_only_text(text):\n",
        "  word_only_text = remove_non_word_chars(text)\n",
        "  extra_space_removed_text = remove_white_spaces(word_only_text)\n",
        "\n",
        "  return extra_space_removed_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6OQQDual4u_E"
      },
      "outputs": [],
      "source": [
        "def get_texts_cer(reference, model_output):\n",
        "  # Preprocess input texts to only contain word characters\n",
        "  word_only_reference = get_word_only_text(reference)\n",
        "  word_only_output = get_word_only_text(model_output)\n",
        "\n",
        "  # Return +infinity for CER if any of the texts is empty\n",
        "  if not word_only_reference.strip() or not word_only_output.strip():\n",
        "    return float('inf')\n",
        "\n",
        "  return cer(word_only_reference, word_only_output)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ncWQnPdW4u_E"
      },
      "outputs": [],
      "source": [
        "def get_avg_cer_of_method(method_outputs, references):\n",
        "  cers = []\n",
        "  for idx, o in enumerate(method_outputs):\n",
        "    cer = get_texts_cer(o.replace('-', ''), references[idx][1].replace('-', ''))\n",
        "    if cer != float('inf'):\n",
        "      cers.append(cer)\n",
        "\n",
        "  return sum(cers) / len(cers)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Homograph Evaluation"
      ],
      "metadata": {
        "id": "oBgNtpFQDwku"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_homograph_performance(outputs, references):\n",
        "  corrects = 0\n",
        "  total = 0\n",
        "\n",
        "  for idx, (g, p, homograph, right) in enumerate(references):\n",
        "    if homograph != '':\n",
        "      total += 1\n",
        "      if right in outputs[idx]:\n",
        "        corrects += 1\n",
        "\n",
        "  return corrects / total"
      ],
      "metadata": {
        "id": "J445ULEvEEDn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Full bench"
      ],
      "metadata": {
        "id": "JGEUIrbi9kNH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "benchmark = []\n",
        "\n",
        "for g, p in mana_evaluation_data:\n",
        "  benchmark.append((g, p, '', ''))\n",
        "\n",
        "for g, p in commonvoice_evaluation_data:\n",
        "  benchmark.append((g, p, '', ''))\n",
        "\n",
        "for g, p, w, r in homograph_evaluation_data:\n",
        "  benchmark.append((g, p, w, r))\n",
        "\n",
        "benchmark = benchmark[:400]"
      ],
      "metadata": {
        "id": "fGzQvL8V9mln"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_all_metrics(predictions):\n",
        "  per = get_avg_cer_of_method(predictions, benchmark) * 100\n",
        "  homograph = get_homograph_performance(predictions, benchmark) * 100\n",
        "\n",
        "  print(f\"PER: \\t\\t\\t{per:.4f}\")\n",
        "  print(f\"HOMOGRAPH: \\t\\t{homograph:.4f}\")"
      ],
      "metadata": {
        "id": "DpSqE5oPbmAy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sentence Inference"
      ],
      "metadata": {
        "id": "_-mKfiwYIMdQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from hazm import WordTokenizer, Normalizer\n",
        "tokenizer = WordTokenizer()\n",
        "normalizer = Normalizer()"
      ],
      "metadata": {
        "id": "PNuCMdIuVPf5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sentence_inference(sent):\n",
        "  phonemes = []\n",
        "  tokens = tokenizer.tokenize(normalizer.normalize(sent))\n",
        "  i = 0\n",
        "  while i < len(tokens):\n",
        "    subsent = ' '.join(tokens[i:i+3])\n",
        "    try:\n",
        "      subphoneme = phonemizer.phonemize(subsent)\n",
        "    except:\n",
        "      subphoneme = ''\n",
        "    phonemes.append(subphoneme)\n",
        "    i += len(tokens[i:i+3])\n",
        "\n",
        "  phoneme = ' '.join(phonemes)\n",
        "  return phoneme"
      ],
      "metadata": {
        "id": "rTPVMXEGU_9r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# outputs"
      ],
      "metadata": {
        "id": "fRaAhTMsMHBJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "from tqdm import tqdm\n",
        "\n",
        "# Initialize variables\n",
        "outputs = []\n",
        "start_time = time.time()  # Start total timer\n",
        "\n",
        "# Process each sample with progress bar\n",
        "for g, p, _, _ in tqdm(benchmark, desc=\"Running inference\"):\n",
        "    o = sentence_inference(g)\n",
        "    outputs.append(o)\n",
        "\n",
        "# Calculate timing stats\n",
        "total_time = time.time() - start_time\n",
        "avg_time = total_time / len(benchmark) if benchmark else 0"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "acc51819-0c98-4eda-a4cc-270f9707e370",
        "id": "PYn9z4GiMHBf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Running inference: 100%|██████████| 400/400 [01:15<00:00,  5.33it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mapped_outputs = []\n",
        "for o in outputs:\n",
        "  mapped = replace_phonetic_characters(o)\n",
        "  mapped_outputs.append(mapped)"
      ],
      "metadata": {
        "id": "90gAxDT-GMhI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_all_metrics(mapped_outputs)\n",
        "print(f\"TOTAL TIME:\\t\\t{total_time:.4f} (s)\")\n",
        "print(f\"AVG TIME:\\t\\t{avg_time:.4f} (s)\")"
      ],
      "metadata": {
        "id": "zP4Tcj285Ij0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5986173e-4d6b-4ff6-b18c-b1e72e3afd83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PER: \t\t\t25.2989\n",
            "HOMOGRAPH: \t\t29.2453\n",
            "TOTAL TIME:\t\t75.0611 (s)\n",
            "AVG TIME:\t\t0.1877 (s)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Runs\n",
        "\n",
        "## First:\n",
        "\n",
        "```\n",
        "PER: \t\t\t25.2111\n",
        "HOMOGRAPH: \t\t29.7170\n",
        "TOTAL TIME:\t\t85.6780 (s)\n",
        "AVG TIME:\t\t0.2142 (s)\n",
        "```\n",
        "\n",
        "## Second\n",
        "\n",
        "```\n",
        "PER: \t\t\t25.1991\n",
        "HOMOGRAPH: \t\t29.7170\n",
        "TOTAL TIME:\t\t41.5983 (s)\n",
        "AVG TIME:\t\t0.1040 (s)\n",
        "```\n",
        "\n",
        "## Third\n",
        "\n",
        "```\n",
        "PER: \t\t\t25.2278\n",
        "HOMOGRAPH: \t\t28.7736\n",
        "TOTAL TIME:\t\t78.4948 (s)\n",
        "AVG TIME:\t\t0.1962 (s)\n",
        "```\n",
        "\n",
        "\n",
        "## Fourth\n",
        "\n",
        "```\n",
        "PER: \t\t\t25.4065\n",
        "HOMOGRAPH: \t\t28.7736\n",
        "TOTAL TIME:\t\t79.8478 (s)\n",
        "AVG TIME:\t\t0.1996 (s)\n",
        "```\n",
        "\n",
        "## Fifth\n",
        "\n",
        "```\n",
        "PER: \t\t\t25.2989\n",
        "HOMOGRAPH: \t\t29.2453\n",
        "TOTAL TIME:\t\t75.0611 (s)\n",
        "AVG TIME:\t\t0.1877 (s)\n",
        "```"
      ],
      "metadata": {
        "id": "5YPY-B2DjHW7"
      }
    }
  ]
}