milad.hosseini
/
Fairness4LLMs-in-fake-news-detection
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1 Commit
1 Branch
Branch: master
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'master'
${ noResults }
Fairness4LLMs-in-fake-news-.../log_processor.py

log_processor.py 6.6KB
Raw Permalink Blame History

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157 
  1. import pandas as pd

  2. import csv

  3. import numpy as np

  4. 

  5. def check_none_validations(data):

  6.     none_group = {}

  7.     for _, row in data.iterrows():

  8.         if row['Ground Truth'] != 0 and row['Ground Truth'] != 1:

  9.             none_group[(row['News'], row['Perspective'])] = row['Ground Truth']

  10.     return none_group

  11. 

  12. def check_inconsistencies(data):

  13. 

  14.     inconsistencies = {}

  15.     for _, row in data.iterrows():

  16.         for prespective in ['Democrat', 'Republican', 'Neutral']:

  17.             if prespective == 'Democrat':

  18.                 if row['Count True Democrat'] != iter and row['Count False Democrat'] != iter:

  19.                     inconsistency_number = min(row['Count True Democrat'], row['Count False Democrat'])

  20.             elif prespective == 'Republican':

  21.                 if row['Count True Republican'] != iter and row['Count False Republican'] != iter:

  22.                     inconsistency_number = min(row['Count True Republican'], row['Count False Republican'])

  23.             elif prespective == 'Neutral':

  24.                 if row['Count True Neutral'] != iter and row['Count False Neutral'] != iter:

  25.                     inconsistency_number = min(row['Count True Neutral'], row['Count False Neutral'])

  26.             if inconsistency_number != 0:

  27.                 inconsistencies[(row['News'], prespective)] = inconsistency_number

  28. 

  29.     return inconsistencies

  30. 

  31. def save_inconsistencies_to_csv(inconsistencies, file_path):

  32.     with open(file_path, mode='w', newline='') as csv_file:

  33.         writer = csv.writer(csv_file)

  34.         

  35.         writer.writerow(['News', 'Perspective', 'Inconsistency Number'])

  36. 

  37.         for (news, perspective), count in inconsistencies.items():

  38.             writer.writerow([news, perspective, count])

  39. 

  40. def save_none_group_to_csv(none_group, file_path):

  41.     with open(file_path, mode='w', newline='') as csv_file:

  42.         writer = csv.writer(csv_file)

  43.         

  44.         writer.writerow(['News', 'Perspective', 'Validations'])

  45. 

  46.         for (news, perspective), count in none_group.items():

  47.             writer.writerow([news, perspective, count])

  48. 

  49. 

  50. # inconsistencies = check_inconsistencies("updated.csv")

  51. # none_group = check_none_validations("explanations.csv")

  52. # save_inconsistencies_to_csv(inconsistencies, "inconsistencies.csv")

  53. # save_none_group_to_csv(none_group, "none_values.csv")

  54. 

  55. import pandas as pd

  56. import numpy as np

  57. 

  58. def compute_confusion_matrices(updated_file, leaning_file, iter):

  59.     df_updated = pd.read_csv(updated_file, delimiter=',')

  60.     df_leaning = pd.read_csv(leaning_file, delimiter=',')

  61.     

  62.     df_updated = df_updated.merge(df_leaning[['News', 'Leaning']], on='News', how='left')

  63.     

  64.     filtered_df = df_updated[df_updated['Leaning'] == 'R']

  65.     

  66.     prob_1_democrat = filtered_df['Count True Democrat'] / iter

  67.     prob_0_democrat = filtered_df['Count False Democrat'] / iter

  68.     prob_1_republican = filtered_df['Count True Republican'] / iter

  69.     prob_0_republican = filtered_df['Count False Republican'] / iter

  70.     prob_1_neutral = filtered_df['Count True Neutral'] / iter

  71.     prob_0_neutral = filtered_df['Count False Neutral'] / iter

  72.     ground_truth = filtered_df['Ground Truth']

  73.     

  74.     def get_confusion_matrix(ground_truth, prob_1, prob_0):

  75.         TP = np.sum(ground_truth * prob_1)

  76.         FP = np.sum((1 - ground_truth) * prob_1)

  77.         FN = np.sum(ground_truth * prob_0)

  78.         TN = np.sum((1 - ground_truth) * prob_0)

  79.         return np.array([[TP, FP], [FN, TN]])

  80.     

  81.     confusion_matrix_prob_democrat = get_confusion_matrix(ground_truth, prob_1_democrat, prob_0_democrat)

  82.     confusion_matrix_prob_republican = get_confusion_matrix(ground_truth, prob_1_republican, prob_0_republican)

  83.     confusion_matrix_prob_neutral = get_confusion_matrix(ground_truth, prob_1_neutral, prob_0_neutral)

  84.     

  85.     return confusion_matrix_prob_democrat, confusion_matrix_prob_republican, confusion_matrix_prob_neutral

  86. 

  87. confusion_matrix_democrat, confusion_matrix_republican, confusion_matrix_neutral = compute_confusion_matrices(

  88.    updated_file='/LLaMa/updated1.csv',

  89.    leaning_file='/news_leaning_dataset.csv',

  90.    iter=10

  91. )

  92. 

  93. print("Confusion Matrix - Democrat:\n", confusion_matrix_democrat)

  94. print("Confusion Matrix - Republican:\n", confusion_matrix_republican)

  95. print("Confusion Matrix - Neutral:\n", confusion_matrix_neutral)

  96. 

  97. 

  98. def report_performance_through_leanings():

  99.     df = pd.read_csv('news_leaning_dataset.csv', delimiter=',')

  100.     

  101.     r001 = []

  102.     d110 = []

  103.     d010 = []

  104.     r101 = []

  105.     

  106.     for _, row in df.iterrows():

  107.         leaning = row['Leaning']

  108.         democrat_response = row['ChatGPT’s response from the perspective of a Conservative (Democrat) viewpoint']

  109.         republican_response = row['ChatGPT’s response from the perspective of a Rdaical (Republican) viewpoint']

  110.         if leaning == 'R' and row['Ground Truth'] == 0 and democrat_response == 0 and republican_response == 1:

  111.             r001.append(row['News'])

  112.         elif leaning == 'D' and row['Ground Truth'] == 1 and democrat_response == 1 and republican_response == 0:

  113.             d110.append(row['News'])

  114.         elif leaning == 'D' and row['Ground Truth'] == 0 and democrat_response == 1 and republican_response == 0:

  115.             d010.append(row['News'])

  116.         elif leaning == 'R' and row['Ground Truth'] == 1 and democrat_response == 0 and republican_response == 1:

  117.             r101.append(row['News'])

  118.     print('_______________________________________________________')

  119.     print('Leaning: R, Ground Truth: 0, Democrat: 0, Republican: 1')

  120.     for news in r001:

  121.         print(news)

  122.     print('_______________________________________________________')

  123.     print('Leaning: D, Ground Truth: 1, Democrat: 1, Republican: 0')

  124.     for news in d110:

  125.         print(news)

  126.     print('_______________________________________________________')

  127.     print('Leaning: D, Ground Truth: 0, Democrat: 1, Republican: 0')

  128.     for news in d010:

  129.         print(news)

  130.     print('_______________________________________________________')

  131.     print('Leaning: R, Ground Truth: 1, Democrat: 0, Republican: 1')

  132.     for news in r101:

  133.         print(news)

  134. 

  135. # report_performance_through_leanings()

  136. df = pd.read_csv('news_leaning_dataset.csv', delimiter=',')

  137. iter = 10

  138. d1 = 0

  139. r1 = 0

  140. d2 = 0

  141. r2 = 0

  142. d3 = 0

  143. r3 = 0

  144. for _, row in df.iterrows():

  145.         if row['Ground Truth'] == 1 and row['Leaning'] == 'N':

  146.             d1 += 1

  147.         elif row['Ground Truth'] == 0 and row['Leaning'] == 'N':

  148.             r1 += 1

  149.         elif row['Ground Truth'] == 1 and row['Leaning'] == 'R':

  150.             d2 += 1

  151.         elif row['Ground Truth'] == 0 and row['Leaning'] == 'R':

  152.             r2 += 1

  153.         elif row['Ground Truth'] == 1 and row['Leaning'] == 'V':

  154.             d3 += 1

  155.         elif row['Ground Truth'] == 0 and row['Leaning'] == 'V':

  156.             r3 += 1

  157. # print(d1, r1, d2, r2, d3, r3)