{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"collapsed_sections": [
"EOZGZa2lMfPe",
"VtxEYym69RUH",
"AdU8VMTIOWLZ",
"a3zuvbqx2l68",
"R6PE5ds45TPr",
"JGEUIrbi9kNH"
]
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "WEY5MiKLzurH"
},
"source": [
"# Setup Environment"
]
},
{
"cell_type": "code",
"source": [
"! pip install PersianG2p"
],
"metadata": {
"id": "JJ9VISNqZS6m",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"outputId": "49fc5536-24cc-44d2-8a9d-47c156afa433"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Collecting PersianG2p\n",
" Downloading PersianG2p-0.3.2-py3-none-any.whl.metadata (6.5 kB)\n",
"Collecting hazm (from PersianG2p)\n",
" Downloading hazm-0.10.0-py3-none-any.whl.metadata (11 kB)\n",
"Requirement already satisfied: inflect in /usr/local/lib/python3.11/dist-packages (from PersianG2p) (7.5.0)\n",
"Collecting num2fawords (from PersianG2p)\n",
" Downloading num2fawords-1.1-py3-none-any.whl.metadata (4.1 kB)\n",
"Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from PersianG2p) (2.0.2)\n",
"Collecting fasttext-wheel<0.10.0,>=0.9.2 (from hazm->PersianG2p)\n",
" Downloading fasttext_wheel-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB)\n",
"Collecting flashtext<3.0,>=2.7 (from hazm->PersianG2p)\n",
" Downloading flashtext-2.7.tar.gz (14 kB)\n",
" Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
"Collecting gensim<5.0.0,>=4.3.1 (from hazm->PersianG2p)\n",
" Downloading gensim-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (8.1 kB)\n",
"Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.11/dist-packages (from hazm->PersianG2p) (3.9.1)\n",
"Collecting numpy (from PersianG2p)\n",
" Downloading numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB)\n",
"Collecting python-crfsuite<0.10.0,>=0.9.9 (from hazm->PersianG2p)\n",
" Downloading python_crfsuite-0.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.3 kB)\n",
"Requirement already satisfied: scikit-learn<2.0.0,>=1.2.2 in /usr/local/lib/python3.11/dist-packages (from hazm->PersianG2p) (1.6.1)\n",
"Requirement already satisfied: more_itertools>=8.5.0 in /usr/local/lib/python3.11/dist-packages (from inflect->PersianG2p) (10.7.0)\n",
"Requirement already satisfied: typeguard>=4.0.1 in /usr/local/lib/python3.11/dist-packages (from inflect->PersianG2p) (4.4.2)\n",
"Collecting pybind11>=2.2 (from fasttext-wheel<0.10.0,>=0.9.2->hazm->PersianG2p)\n",
" Downloading pybind11-2.13.6-py3-none-any.whl.metadata (9.5 kB)\n",
"Requirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from fasttext-wheel<0.10.0,>=0.9.2->hazm->PersianG2p) (75.2.0)\n",
"Collecting scipy<1.14.0,>=1.7.0 (from gensim<5.0.0,>=4.3.1->hazm->PersianG2p)\n",
" Downloading scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.11/dist-packages (from gensim<5.0.0,>=4.3.1->hazm->PersianG2p) (7.1.0)\n",
"Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm->PersianG2p) (8.1.8)\n",
"Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm->PersianG2p) (1.4.2)\n",
"Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm->PersianG2p) (2024.11.6)\n",
"Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from nltk<4.0.0,>=3.8.1->hazm->PersianG2p) (4.67.1)\n",
"Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn<2.0.0,>=1.2.2->hazm->PersianG2p) (3.6.0)\n",
"Requirement already satisfied: typing_extensions>=4.10.0 in /usr/local/lib/python3.11/dist-packages (from typeguard>=4.0.1->inflect->PersianG2p) (4.13.2)\n",
"Requirement already satisfied: wrapt in /usr/local/lib/python3.11/dist-packages (from smart-open>=1.8.1->gensim<5.0.0,>=4.3.1->hazm->PersianG2p) (1.17.2)\n",
"Downloading PersianG2p-0.3.2-py3-none-any.whl (928 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m928.1/928.1 kB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading hazm-0.10.0-py3-none-any.whl (892 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m892.6/892.6 kB\u001b[0m \u001b[31m31.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m31.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading num2fawords-1.1-py3-none-any.whl (9.8 kB)\n",
"Downloading fasttext_wheel-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.4 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m55.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading gensim-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (26.7 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.7/26.7 MB\u001b[0m \u001b[31m46.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading python_crfsuite-0.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m38.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading pybind11-2.13.6-py3-none-any.whl (243 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m243.3/243.3 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading scipy-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (38.6 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.6/38.6 MB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hBuilding wheels for collected packages: flashtext\n",
" Building wheel for flashtext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
" Created wheel for flashtext: filename=flashtext-2.7-py2.py3-none-any.whl size=9300 sha256=bbdfef47ecf07814b056f2477b78913068fd94f3b79fbe865726138fe5a4ddc1\n",
" Stored in directory: /root/.cache/pip/wheels/49/20/47/f03dfa8a7239c54cbc44ff7389eefbf888d2c1873edaaec888\n",
"Successfully built flashtext\n",
"Installing collected packages: num2fawords, flashtext, python-crfsuite, pybind11, numpy, scipy, fasttext-wheel, gensim, hazm, PersianG2p\n",
" Attempting uninstall: numpy\n",
" Found existing installation: numpy 2.0.2\n",
" Uninstalling numpy-2.0.2:\n",
" Successfully uninstalled numpy-2.0.2\n",
" Attempting uninstall: scipy\n",
" Found existing installation: scipy 1.15.2\n",
" Uninstalling scipy-1.15.2:\n",
" Successfully uninstalled scipy-1.15.2\n",
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
"blosc2 3.3.2 requires numpy>=1.26, but you have numpy 1.24.3 which is incompatible.\n",
"thinc 8.3.6 requires numpy<3.0.0,>=2.0.0, but you have numpy 1.24.3 which is incompatible.\n",
"treescope 0.1.9 requires numpy>=1.25.2, but you have numpy 1.24.3 which is incompatible.\n",
"pymc 5.22.0 requires numpy>=1.25.0, but you have numpy 1.24.3 which is incompatible.\n",
"albumentations 2.0.6 requires numpy>=1.24.4, but you have numpy 1.24.3 which is incompatible.\n",
"albucore 0.0.24 requires numpy>=1.24.4, but you have numpy 1.24.3 which is incompatible.\n",
"tensorflow 2.18.0 requires numpy<2.1.0,>=1.26.0, but you have numpy 1.24.3 which is incompatible.\n",
"jax 0.5.2 requires numpy>=1.25, but you have numpy 1.24.3 which is incompatible.\n",
"jaxlib 0.5.1 requires numpy>=1.25, but you have numpy 1.24.3 which is incompatible.\u001b[0m\u001b[31m\n",
"\u001b[0mSuccessfully installed PersianG2p-0.3.2 fasttext-wheel-0.9.2 flashtext-2.7 gensim-4.3.3 hazm-0.10.0 num2fawords-1.1 numpy-1.24.3 pybind11-2.13.6 python-crfsuite-0.9.11 scipy-1.13.1\n"
]
},
{
"output_type": "display_data",
"data": {
"application/vnd.colab-display-data+json": {
"pip_warning": {
"packages": [
"numpy"
]
},
"id": "64c42c29e8e04317a2d190a1cd236293"
}
},
"metadata": {}
}
]
},
{
"cell_type": "code",
"source": [
"! pip install jiwer"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "u-Wc6za7ejMs",
"outputId": "1a08f250-f5a8-4d78-a476-a0196241694c"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Collecting jiwer\n",
" Downloading jiwer-3.1.0-py3-none-any.whl.metadata (2.6 kB)\n",
"Requirement already satisfied: click>=8.1.8 in /usr/local/lib/python3.11/dist-packages (from jiwer) (8.1.8)\n",
"Collecting rapidfuzz>=3.9.7 (from jiwer)\n",
" Downloading rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
"Downloading jiwer-3.1.0-py3-none-any.whl (22 kB)\n",
"Downloading rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m23.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hInstalling collected packages: rapidfuzz, jiwer\n",
"Successfully installed jiwer-3.1.0 rapidfuzz-3.13.0\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"import re\n",
"from tqdm import tqdm\n",
"import csv\n",
"import pandas as pd\n",
"import json\n",
"import itertools\n",
"from jiwer import cer"
],
"metadata": {
"id": "xDqy82HJeoui"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Setup Model"
],
"metadata": {
"id": "Y3BIUX2depmo"
}
},
{
"cell_type": "code",
"source": [
"from PersianG2p import Persian_g2p_converter\n",
"\n",
"PersianG2Pconverter = Persian_g2p_converter()"
],
"metadata": {
"id": "dOhI9I0TZPX-"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"PersianG2Pconverter.transliterate('دلم میخواست برم ', tidy = False, secret = True)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"id": "K2tlspKhZz7m",
"outputId": "3e42869a-1b12-4c20-cab9-1cbda35711ca"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'dalam mixAst beram'"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
}
},
"metadata": {},
"execution_count": 4
}
]
},
{
"cell_type": "code",
"source": [
"PersianG2Pconverter.transliterate('دلم میخواست برم ', tidy = True, secret = True)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"id": "qhZVmJ1hZ7V4",
"outputId": "dcbfa84e-97e3-4ec9-b906-5f29be545c11"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'dalam mixāst beram'"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
}
},
"metadata": {},
"execution_count": 5
}
]
},
{
"cell_type": "code",
"source": [
"PersianG2Pconverter.transliterate('انجمن نابینایان برای افرادی که تمایل به شنیدن مجلهی نسل مانا را دارند، این امکان را فراهم کردهاست.', tidy = False, secret = True)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"id": "24-8fAuZaAxf",
"outputId": "a1d4d066-e1ac-4e35-85fb-fbce079cd422"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'anjoman nAbinA?An barA^ye efrAdi ke tamAyol be Senidan majele?i nasl mAnA rA dArand ، in emkAn rA farAham kerdedest .'"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
}
},
"metadata": {},
"execution_count": 6
}
]
},
{
"cell_type": "markdown",
"source": [
"# mapping"
],
"metadata": {
"id": "VtxEYym69RUH"
}
},
{
"cell_type": "code",
"source": [
"output_to_phonetics_map = {\n",
" 'м': 'm',\n",
" 'ʷ':' v',\n",
" 'w': 'v',\n",
" 'c': 'k',\n",
" 'ĉ': 'C',\n",
" 'č': 'C',\n",
" '̕': \"?\",\n",
" \"'\": '?',\n",
" 'ʔ': \"?\",\n",
" 'ꞌ': \"?\",\n",
" '̛': \"?\",\n",
" '’': \"?\",\n",
" 'ʼ': \"?\",\n",
" \"'\": '?',\n",
" 'â': 'A',\n",
" 'â': 'A',\n",
" 'ȃ': 'A',\n",
" 'ž': 'Z',\n",
" 'š': 'S',\n",
" 'W': 'v',\n",
" 'β': 'f',\n",
" 'е': 'e',\n",
" '`': \"?\",\n",
" 'ɑ': 'A',\n",
" 'ɑ': 'A',\n",
" 'ʃ': 'S',\n",
" 'ð': 'z',\n",
" 'ɾ': 'r',\n",
" 'æ': 'a',\n",
" 'ɪ': 'e',\n",
" 'χ': 'x',\n",
" 'ɣ': 'q',\n",
" 'ʒ': 'Z',\n",
" ':': '',\n",
" 'ː': '',\n",
" 'ā': 'A',\n",
" 'ː': '',\n",
" 'ä': 'A',\n",
" 'á': 'A',\n",
" 'š': 'S',\n",
" 'ū': 'u',\n",
" 'û': 'u',\n",
" 'ś': 's',\n",
" 'ī': 'i',\n",
" 'í': 'i',\n",
" 'î': 'i',\n",
" 'é': 'e',\n",
" 'ḥ': 'h',\n",
" 'ɒ': 'A',\n",
" 'ʰ': '',\n",
" 'ə': 'e',\n",
" 'R': 'r',\n",
" 'W': 'v',\n",
" 'Q': 'q',\n",
" 'T': 't',\n",
" 'Y': 'y',\n",
" 'P': 'p',\n",
" 'D': 'd',\n",
" 'F': 'f',\n",
" 'H': 'h',\n",
" 'J': 'j',\n",
" 'L': 'l',\n",
" 'X': 'x',\n",
" 'V': 'v',\n",
" 'B': 'b',\n",
" 'N': 'n',\n",
" 'M': 'm',\n",
" 'K': 'k',\n",
" 'G': 'g',\n",
" 'U': 'u',\n",
" 'O': 'o',\n",
" 'I': 'i',\n",
" 'E': 'e',\n",
" 'ŋ': 'ng',\n",
" '.': '',\n",
" 'ɛ': 'e',\n",
" 'ʊ': 'u',\n",
" \"ˈ\": '?',\n",
" 'ù': 'u',\n",
" 'θ': 's',\n",
" '̪': '',\n",
" 'ũ': 'u',\n",
" '_': '',\n",
" 'ç': 'C',\n",
" 'ĝ': 'q',\n",
" 'ɢ': 'q',\n",
" 'ː': '',\n",
" 'í': 'i',\n",
" 'ŝ': 'S',\n",
" '!': '',\n",
" 'ǧ': 'q',\n",
" 'ʻ': '?',\n",
" 'è': 'e',\n",
" '�': '',\n",
" 'ú': 'u',\n",
" 'ô': 'o',\n",
" 'ē': 'e',\n",
" 'à': 'A',\n",
" 'ă': 'A',\n",
" 'ǐ': 'i',\n",
" 'ü': 'u',\n",
" '\\u200e': '',\n",
" 'ğ': 'q',\n",
" 'ṣ': 'S',\n",
" 'â': 'A',\n",
" 'â': 'A',\n",
" 'ȃ': 'A',\n",
" 'ž': 'Z',\n",
" 'š': 'S',\n",
" 'ā': 'A',\n",
" 'ː': '',\n",
" 'ä': 'A',\n",
" 'á': 'A',\n",
" 'š': 'S',\n",
" 'ū': 'u',\n",
" 'û': 'u',\n",
" 'ś': 'S',\n",
" 'ī': 'i',\n",
" 'í': 'i',\n",
" 'î': 'i',\n",
" 'é': 'e',\n",
"}\n",
"\n",
"consonants_regex = '(?=' + '|'.join(['q', 'r', 't', 'y', 'p', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'z', 'x', 'c', 'v', 'b', 'n', 'm', 'Q', 'R', 'T', 'Y', 'P', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L', 'Z', 'X', 'C', 'V', 'B', 'N', 'M' ]) + ')'\n",
"vowels_regex = '(?=' + '|'.join(['a', 'A', 'e', 'i', 'u', 'o']) + ')'\n",
"\n",
"\n",
"def replace_phonetic_characters(input_string, char_map=output_to_phonetics_map, from_phonetics=False):\n",
" substituted = re.sub(r'tʃʰ', 'C', input_string)\n",
" substituted = re.sub(r't͡ʃ', 'C', input_string)\n",
" substituted = re.sub(r'tʃ', 'C', substituted)\n",
" substituted = re.sub(r't͡S', 'C', substituted)\n",
" substituted = re.sub(r'ow', 'o', substituted)\n",
" substituted = re.sub('d͡ʒ', 'j', substituted)\n",
" substituted = re.sub('dʒ', 'j', substituted)\n",
"\n",
" # Create a translation table using str.maketrans\n",
" translation_table = str.maketrans(char_map)\n",
"\n",
" # Use str.translate to replace characters based on the translation table\n",
" translated = substituted.translate(translation_table)\n",
"\n",
" return translated"
],
"metadata": {
"id": "TKx8oA1n7rKh"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "XjAPkfq7SF87"
},
"source": [
"# Get Evaluation Data"
]
},
{
"cell_type": "code",
"source": [
"!wget https://huggingface.co/datasets/MahtaFetrat/SentenceBench/raw/main/SentenceBench.csv"
],
"metadata": {
"id": "qwCG0jX-88nQ",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "873c1f69-e2b4-4363-c7a5-f38b47513659"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"--2025-05-10 11:45:09-- https://huggingface.co/datasets/MahtaFetrat/SentenceBench/raw/main/SentenceBench.csv\n",
"Resolving huggingface.co (huggingface.co)... 13.226.52.8, 13.226.52.35, 13.226.52.100, ...\n",
"Connecting to huggingface.co (huggingface.co)|13.226.52.8|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 56026 (55K) [text/plain]\n",
"Saving to: ‘SentenceBench.csv’\n",
"\n",
"\rSentenceBench.csv 0%[ ] 0 --.-KB/s \rSentenceBench.csv 100%[===================>] 54.71K --.-KB/s in 0.01s \n",
"\n",
"2025-05-10 11:45:09 (4.13 MB/s) - ‘SentenceBench.csv’ saved [56026/56026]\n",
"\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"sentence_bench = pd.read_csv('SentenceBench.csv')"
],
"metadata": {
"id": "hJO-UAPDQvcb"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"sentence_bench.head(3)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 143
},
"id": "qlYbrnUa9LAN",
"outputId": "e27fedf6-570c-4a31-8d6b-84c64b23dd90"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" dataset grapheme \\\n",
"0 homograph من قدر تو را میدانم \n",
"1 homograph از قضای الهی به قدر الهی پناه میبرم \n",
"2 homograph به دست و صورتم کرم زدم \n",
"\n",
" phoneme homograph word \\\n",
"0 man qadr-e to rA mi-dAnam قدر \n",
"1 ?az qazAy ?elAhi be qadar-e ?elAhi panAh mi-baram قدر \n",
"2 be dast-o suratam kerem zadam کرم \n",
"\n",
" pronunciation \n",
"0 qadr \n",
"1 qadar \n",
"2 kerem "
],
"text/html": [
"\n",
" <div id=\"df-2f9a2ff8-5fad-4b3d-a3e6-12372061610b\" class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>dataset</th>\n",
" <th>grapheme</th>\n",
" <th>phoneme</th>\n",
" <th>homograph word</th>\n",
" <th>pronunciation</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>homograph</td>\n",
" <td>من قدر تو را میدانم</td>\n",
" <td>man qadr-e to rA mi-dAnam</td>\n",
" <td>قدر</td>\n",
" <td>qadr</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>homograph</td>\n",
" <td>از قضای الهی به قدر الهی پناه میبرم</td>\n",
" <td>?az qazAy ?elAhi be qadar-e ?elAhi panAh mi-baram</td>\n",
" <td>قدر</td>\n",
" <td>qadar</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>homograph</td>\n",
" <td>به دست و صورتم کرم زدم</td>\n",
" <td>be dast-o suratam kerem zadam</td>\n",
" <td>کرم</td>\n",
" <td>kerem</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <div class=\"colab-df-buttons\">\n",
"\n",
" <div class=\"colab-df-container\">\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2f9a2ff8-5fad-4b3d-a3e6-12372061610b')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" .colab-df-buttons div {\n",
" margin-bottom: 4px;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-2f9a2ff8-5fad-4b3d-a3e6-12372061610b button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-2f9a2ff8-5fad-4b3d-a3e6-12372061610b');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
"\n",
"\n",
" <div id=\"df-de5a023e-64df-4224-8df0-57191f788f74\">\n",
" <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-de5a023e-64df-4224-8df0-57191f788f74')\"\n",
" title=\"Suggest charts\"\n",
" style=\"display:none;\">\n",
"\n",
"<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
" width=\"24px\">\n",
" <g>\n",
" <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
" </g>\n",
"</svg>\n",
" </button>\n",
"\n",
"<style>\n",
" .colab-df-quickchart {\n",
" --bg-color: #E8F0FE;\n",
" --fill-color: #1967D2;\n",
" --hover-bg-color: #E2EBFA;\n",
" --hover-fill-color: #174EA6;\n",
" --disabled-fill-color: #AAA;\n",
" --disabled-bg-color: #DDD;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-quickchart {\n",
" --bg-color: #3B4455;\n",
" --fill-color: #D2E3FC;\n",
" --hover-bg-color: #434B5C;\n",
" --hover-fill-color: #FFFFFF;\n",
" --disabled-bg-color: #3B4455;\n",
" --disabled-fill-color: #666;\n",
" }\n",
"\n",
" .colab-df-quickchart {\n",
" background-color: var(--bg-color);\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: var(--fill-color);\n",
" height: 32px;\n",
" padding: 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-quickchart:hover {\n",
" background-color: var(--hover-bg-color);\n",
" box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: var(--button-hover-fill-color);\n",
" }\n",
"\n",
" .colab-df-quickchart-complete:disabled,\n",
" .colab-df-quickchart-complete:disabled:hover {\n",
" background-color: var(--disabled-bg-color);\n",
" fill: var(--disabled-fill-color);\n",
" box-shadow: none;\n",
" }\n",
"\n",
" .colab-df-spinner {\n",
" border: 2px solid var(--fill-color);\n",
" border-color: transparent;\n",
" border-bottom-color: var(--fill-color);\n",
" animation:\n",
" spin 1s steps(1) infinite;\n",
" }\n",
"\n",
" @keyframes spin {\n",
" 0% {\n",
" border-color: transparent;\n",
" border-bottom-color: var(--fill-color);\n",
" border-left-color: var(--fill-color);\n",
" }\n",
" 20% {\n",
" border-color: transparent;\n",
" border-left-color: var(--fill-color);\n",
" border-top-color: var(--fill-color);\n",
" }\n",
" 30% {\n",
" border-color: transparent;\n",
" border-left-color: var(--fill-color);\n",
" border-top-color: var(--fill-color);\n",
" border-right-color: var(--fill-color);\n",
" }\n",
" 40% {\n",
" border-color: transparent;\n",
" border-right-color: var(--fill-color);\n",
" border-top-color: var(--fill-color);\n",
" }\n",
" 60% {\n",
" border-color: transparent;\n",
" border-right-color: var(--fill-color);\n",
" }\n",
" 80% {\n",
" border-color: transparent;\n",
" border-right-color: var(--fill-color);\n",
" border-bottom-color: var(--fill-color);\n",
" }\n",
" 90% {\n",
" border-color: transparent;\n",
" border-bottom-color: var(--fill-color);\n",
" }\n",
" }\n",
"</style>\n",
"\n",
" <script>\n",
" async function quickchart(key) {\n",
" const quickchartButtonEl =\n",
" document.querySelector('#' + key + ' button');\n",
" quickchartButtonEl.disabled = true; // To prevent multiple clicks.\n",
" quickchartButtonEl.classList.add('colab-df-spinner');\n",
" try {\n",
" const charts = await google.colab.kernel.invokeFunction(\n",
" 'suggestCharts', [key], {});\n",
" } catch (error) {\n",
" console.error('Error during call to suggestCharts:', error);\n",
" }\n",
" quickchartButtonEl.classList.remove('colab-df-spinner');\n",
" quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
" }\n",
" (() => {\n",
" let quickchartButtonEl =\n",
" document.querySelector('#df-de5a023e-64df-4224-8df0-57191f788f74 button');\n",
" quickchartButtonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
" })();\n",
" </script>\n",
" </div>\n",
" </div>\n",
" </div>\n"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "dataframe",
"variable_name": "sentence_bench",
"summary": "{\n \"name\": \"sentence_bench\",\n \"rows\": 400,\n \"fields\": [\n {\n \"column\": \"dataset\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"homograph\",\n \"mana-tts\",\n \"commonvoice\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"grapheme\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 400,\n \"samples\": [\n \"\\u0622\\u06cc\\u0627 \\u0628\\u0627\\u06cc\\u062f \\u062d\\u0642\\u06cc\\u0642\\u062a \\u0631\\u0627 \\u0628\\u0647 \\u0622\\u0646\\u200c\\u0647\\u0627 \\u0628\\u06af\\u0648\\u06cc\\u06cc\\u0645\\u061f\",\n \"\\u06a9\\u0647 \\u067e\\u06cc\\u0634 \\u0627\\u0632 \\u0627\\u0646\\u0642\\u0644\\u0627\\u0628 \\u0628\\u0647 \\u062e\\u0648\\u0627\\u0628\\u06af\\u0627\\u0647 \\u062f\\u062e\\u062a\\u0631\\u0627\\u0646 \\u0648 \\u0632\\u0646\\u0627\\u0646 \\u0646\\u0627\\u0628\\u06cc\\u0646\\u0627 \\u0627\\u062e\\u062a\\u0635\\u0627\\u0635\\u200c\\u06cc\\u0627\\u0641\\u062a\\u0647 \\u0628\\u0648\\u062f. \\u0627\\u063a\\u0644\\u0628 \\u0632\\u0646\\u0627\\u0646\\u06cc \\u06a9\\u0647 \\u062f\\u0631 \\u0627\\u06cc\\u0646 \\u062e\\u0648\\u0627\\u0628\\u06af\\u0627\\u0647 \\u0632\\u0646\\u062f\\u06af\\u06cc \\u0645\\u06cc\\u200c\\u06a9\\u0631\\u062f\\u0646\\u062f\\u060c \",\n \"\\u062f\\u0648\\u062f \\u0648 \\u0645\\u0647 \\u063a\\u0644\\u06cc\\u0638\\u06cc \\u062f\\u0631 \\u0645\\u062d\\u06cc\\u0637 \\u067e\\u06cc\\u0686\\u06cc\\u062f\\u0647 \\u0628\\u0648\\u062f\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"phoneme\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 400,\n \"samples\": [\n \"?AyA bAyad haqiqat rA be ?AnhA beguyim\\u061f\",\n \"ke piS ?az ?enqelAb be xAbgAh-e doxtarAn va zanAn-e nAbinA ?extesAsyAfte bud ?aqlab-e zanAni ke dar ?in xAbgAh zendegi mikardand\",\n \"dud-o meh-e qalizi dar mohit piCide bud\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"homograph word\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 101,\n \"samples\": [\n \"\\u06af\\u0631\\u06cc\\u0645\",\n \"\\u0633\\u0628\\u06a9\\u06cc\",\n \"\\u06a9\\u0645\\u06cc\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"pronunciation\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 210,\n \"samples\": [\n \"darham\",\n \"Sum\",\n \"moSk\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
}
},
"metadata": {},
"execution_count": 10
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "wDV7ysXf2b_H"
},
"source": [
"### Get ManaTTS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "TcL5ZLvSSnVB",
"outputId": "40a6c0bc-e916-4515-ae86-158135f8242d"
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[('در این نوشته بنا داریم با یک ابزار ساده و مکانیکی افزایش بینایی برای افراد کم\\u200cبینا ',\n",
" 'dar ?in neveSte banA dArim bA yek ?abzAr-e sAde va mekAniki-ye ?afzAyeS-e binAyi barAye ?afrAd-e kam\\u200cbinA '),\n",
" ('به نام بی\\u200cوپتیک یا عدسی دورنما آشنا شویم. ',\n",
" 'be nAm-e biyoptik yA ?adasi-ye durnamA ?ASnA Savim'),\n",
" ('دراین\\u200cصورت، انجام خودارزیابی و ارائه بازخورد بر عهده خودتان است. ',\n",
" 'dar ?in surat ?anjAm-e xod?arzyAbi va ?erA?e-ye bAzxord bar ?ohde-ye xodetAn ?ast ')]"
]
},
"metadata": {},
"execution_count": 11
}
],
"source": [
"filtered_rows = sentence_bench[sentence_bench['dataset'] == 'mana-tts'][['grapheme', 'phoneme']]\n",
"\n",
"# Convert to a list of tuples\n",
"mana_evaluation_data = list(filtered_rows.itertuples(index=False, name=None))\n",
"\n",
"mana_evaluation_data[:3]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Jjacw9Mp2eoX"
},
"source": [
"### Get CommonVoice"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "-yQnqCGw26sk",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "d45bb201-06b7-48c5-d153-d6908de6bb2f"
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[('در اکثر شهرها، مرکزی برای خرید دوچرخه وجود دارد.',\n",
" 'dar ?aksar-e Sahr-hA, markazi barAye xarid-e doCarxe vojud dArad.'),\n",
" ('پس از مدرسه کودکان به سوی خانه جست و خیز کردند.',\n",
" 'pas ?az madrese kudakAn be suye xAne jast-o-xiz kardand.'),\n",
" ('شما نگران زن و بچه این نباش.', 'SomA negarAn-e zan-o-baCCe-ye ?in nabAS.')]"
]
},
"metadata": {},
"execution_count": 12
}
],
"source": [
"filtered_rows = sentence_bench[sentence_bench['dataset'] == 'commonvoice'][['grapheme', 'phoneme']]\n",
"\n",
"# Convert to a list of tuples\n",
"commonvoice_evaluation_data = list(filtered_rows.itertuples(index=False, name=None))\n",
"\n",
"commonvoice_evaluation_data[:3]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "ciSPyhRc3Rvo"
},
"source": [
"### Get Homograph"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "XlFc5JbN3Rvz",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "4da90100-f3ef-47b4-ad30-98307394d8d9"
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[('من قدر تو را می\\u200cدانم', 'man qadr-e to rA mi-dAnam', 'قدر', 'qadr'),\n",
" ('از قضای الهی به قدر الهی پناه می\\u200cبرم',\n",
" '?az qazAy ?elAhi be qadar-e ?elAhi panAh mi-baram',\n",
" 'قدر',\n",
" 'qadar'),\n",
" ('به دست و صورتم کرم زدم', 'be dast-o suratam kerem zadam', 'کرم', 'kerem')]"
]
},
"metadata": {},
"execution_count": 13
}
],
"source": [
"filtered_rows = sentence_bench[sentence_bench['dataset'] == 'homograph'][['grapheme', 'phoneme', 'homograph word',\t'pronunciation']]\n",
"\n",
"# Convert to a list of tuples\n",
"homograph_evaluation_data = list(filtered_rows.itertuples(index=False, name=None))\n",
"\n",
"homograph_evaluation_data[:3]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "R6PE5ds45TPr"
},
"source": [
"# Evaluate Method Outputs"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "CLKaERek4u_D"
},
"source": [
"## PER Evaluation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "nBee9xG54u_E"
},
"outputs": [],
"source": [
"def remove_non_word_chars(text):\n",
" pattern = r'[^\\w\\s\\?]'\n",
" cleaned_text = re.sub(pattern, ' ', text)\n",
" return cleaned_text"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "W8PoNV9V4u_E"
},
"outputs": [],
"source": [
"def remove_white_spaces(text):\n",
" cleaned_text = re.sub(r'\\s+', ' ', text)\n",
" return cleaned_text.strip()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "YD0cvnn74u_E"
},
"outputs": [],
"source": [
"def get_word_only_text(text):\n",
" word_only_text = remove_non_word_chars(text)\n",
" extra_space_removed_text = remove_white_spaces(word_only_text)\n",
"\n",
" return extra_space_removed_text"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "6OQQDual4u_E"
},
"outputs": [],
"source": [
"def get_texts_cer(reference, model_output):\n",
" # Preprocess input texts to only contain word characters\n",
" word_only_reference = get_word_only_text(reference)\n",
" word_only_output = get_word_only_text(model_output)\n",
"\n",
" # Return +infinity for CER if any of the texts is empty\n",
" if not word_only_reference.strip() or not word_only_output.strip():\n",
" return float('inf')\n",
"\n",
" return cer(word_only_reference, word_only_output)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "ncWQnPdW4u_E"
},
"outputs": [],
"source": [
"def get_avg_cer_of_method(method_outputs, references):\n",
" cers = []\n",
" for idx, o in enumerate(method_outputs):\n",
" cer = get_texts_cer(o.replace('-', ''), references[idx][1].replace('-', ''))\n",
" if cer != float('inf'):\n",
" cers.append(cer)\n",
"\n",
" return sum(cers) / len(cers)"
]
},
{
"cell_type": "markdown",
"source": [
"## Homograph Evaluation"
],
"metadata": {
"id": "oBgNtpFQDwku"
}
},
{
"cell_type": "code",
"source": [
"def get_homograph_performance(outputs, references):\n",
" corrects = 0\n",
" total = 0\n",
"\n",
" for idx, (g, p, homograph, right) in enumerate(references):\n",
" if homograph != '':\n",
" total += 1\n",
" if right in outputs[idx]:\n",
" corrects += 1\n",
"\n",
" return corrects / total"
],
"metadata": {
"id": "J445ULEvEEDn"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Full bench"
],
"metadata": {
"id": "JGEUIrbi9kNH"
}
},
{
"cell_type": "code",
"source": [
"benchmark = []\n",
"\n",
"for g, p in mana_evaluation_data:\n",
" benchmark.append((g, p, '', ''))\n",
"\n",
"for g, p in commonvoice_evaluation_data:\n",
" benchmark.append((g, p, '', ''))\n",
"\n",
"for g, p, w, r in homograph_evaluation_data:\n",
" benchmark.append((g, p, w, r))\n",
"\n",
"benchmark = benchmark[:400]"
],
"metadata": {
"id": "fGzQvL8V9mln"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"def print_all_metrics(predictions):\n",
" per = get_avg_cer_of_method(predictions, benchmark) * 100\n",
" homograph = get_homograph_performance(predictions, benchmark) * 100\n",
"\n",
" print(f\"PER: \\t\\t\\t{per:.4f}\")\n",
" print(f\"HOMOGRAPH: \\t\\t{homograph:.4f}\")"
],
"metadata": {
"id": "DpSqE5oPbmAy"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# outputs"
],
"metadata": {
"id": "Y8oTYWSXJvup"
}
},
{
"cell_type": "code",
"source": [
"from tqdm import tqdm\n",
"import time\n",
"\n",
"outputs = []\n",
"start_time = time.time()\n",
"\n",
"for g, p, _, _ in tqdm(benchmark):\n",
" o = PersianG2Pconverter.transliterate(g, tidy=False, secret=True)\n",
" outputs.append(o)\n",
"\n",
"total_time = time.time() - start_time\n",
"avg_time = total_time / len(benchmark) if len(benchmark) > 0 else 0"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ECW_8Ja5g7FY",
"outputId": "cb97adcb-d0ff-4151-e8d7-a452a5825d7d"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"100%|██████████| 400/400 [15:20<00:00, 2.30s/it]\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"mapped_outputs = []\n",
"for o in outputs:\n",
" mapped = replace_phonetic_characters(o)\n",
" mapped_outputs.append(mapped)"
],
"metadata": {
"id": "90gAxDT-GMhI"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"print_all_metrics(mapped_outputs)\n",
"print(f\"TOTAL TIME:\\t\\t{total_time:.4f} (s)\")\n",
"print(f\"AVG TIME:\\t\\t{avg_time:.4f} (s)\")"
],
"metadata": {
"id": "zP4Tcj285Ij0",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "619843e5-bb25-4b1b-f062-09a9d1c11f6f"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"PER: \t\t\t15.0414\n",
"HOMOGRAPH: \t\t37.7358\n",
"TOTAL TIME:\t\t920.0853 (s)\n",
"AVG TIME:\t\t2.3002 (s)\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Runs\n",
"\n",
"## First:\n",
"\n",
"```\n",
"PER: \t\t\t15.0414\n",
"homograph: \t\t37.7358\n",
"TOTAL TIME:\t\t874.3154 (s)\n",
"AVG TIME:\t\t2.1858 (s)\n",
"```\n",
"\n",
"## Second\n",
"\n",
"```\n",
"PER: \t\t\t15.0414\n",
"homograph: \t\t37.7358\n",
"TOTAL TIME:\t\t814.6596 (s)\n",
"AVG TIME:\t\t2.0366 (s)\n",
"```\n",
"\n",
"## Third\n",
"\n",
"```\n",
"PER: \t\t\t15.0414\n",
"POLYPHONE: \t\t37.7358\n",
"TOTAL TIME:\t\t845.8805 (s)\n",
"AVG TIME:\t\t2.1147 (s)\n",
"```\n",
"\n",
"## Fourth\n",
"\n",
"```\n",
"PER: \t\t\t15.0414\n",
"HOMOGRAPH: \t\t37.7358\n",
"TOTAL TIME:\t\t882.1829 (s)\n",
"AVG TIME:\t\t2.2055 (s)\n",
"```\n",
"\n",
"## Fifth\n",
"\n",
"```\n",
"PER: \t\t\t15.0414\n",
"HOMOGRAPH: \t\t37.7358\n",
"TOTAL TIME:\t\t920.0853 (s)\n",
"AVG TIME:\t\t2.3002 (s)\n",
"```"
],
"metadata": {
"id": "3vq1em5ElE_Q"
}
}
]
}