Commit 355e49a1 authored by Prathisha Kunnumbrath Manden's avatar Prathisha Kunnumbrath Manden
Browse files

Add std to agent comparison plots

parent cd1e9c30
import os
import numpy as np
import pandas as pd
import scikit_posthocs as sp
from scipy.stats import friedmanchisquare
df = pd.read_csv('exp_results_compare_agents.csv', usecols=["Grid Size", "Number of Colors", "Agent", "Total score per game"])
max_colors = df['Number of Colors'].unique().tolist()
grid_sizes = ['(5, 5)', '(10, 10)', '(15, 15)', '(20, 20)']
agents = ["top_agent", "bottom_agent", "random_agent", "RL Agent(PPO)"]
step = 50
for grid_size in grid_sizes:
labels = []
(row, col) = grid_size.split(',')[0].replace("(", ""), grid_size.split(",")[1].replace(" ", "").replace(")", "")
for color in max_colors:
data_1 = df[(df['Grid Size'] == grid_size) & (df['Number of Colors'] == color) & (df['Agent'] == "top_agent")]
data_2 = df[(df['Grid Size'] == grid_size) & (df['Number of Colors'] == color) & (df['Agent'] == "bottom_agent")]
data_3 = df[(df['Grid Size'] == grid_size) & (df['Number of Colors'] == color) & (df['Agent'] == "random_agent")]
data_4 = df[(df['Grid Size'] == grid_size) & (df['Number of Colors'] == color) & (df['Agent'] == "RL Agent(PPO)")]
if not data_1.empty and not data_2.empty and not data_3.empty and not data_4.empty:
data1 = data_1['Total score per game'].copy().reset_index(drop=True)
df_1 = data1.groupby(data1.index // step).mean().tolist()
# del df_1[20:]
data2 = data_2['Total score per game'].copy().reset_index(drop=True)
df_2 = data2.groupby(data2.index // step).mean().tolist()
# del df_2[20:]
data3 = data_3['Total score per game'].copy().reset_index(drop=True)
df_3 = data3.groupby(data3.index // step).mean().tolist()
# del df_3[20:]
data4 = data_4['Total score per game'].copy().reset_index(drop=True)
df_4 = data4.groupby(data4.index // step).mean().tolist()
# del df_4[20:]
# -----------------------Friedman Test-------------------
# compare samples
print(friedmanchisquare(df_1, df_2, df_3, df_4))
stat, p = friedmanchisquare(df_1, df_2, df_3, df_4)
file_exists = os.path.isfile("stat_significance_agent_comp.csv")
# with open('stat_significance_agent_comp.csv', 'a+', newline='') as csv_file:
# fieldnames = ['Grid Size',
# 'Number of Colors',
# 'statistics',
# 'p-value',
# 'Top agent Mean score',
# 'Top agent Median score',
# 'Top agent Std',
# 'Bottom agent Mean score',
# 'Bottom agent Median score',
# 'Bottom agent Std',
# 'Random agent Mean score',
# 'Random agent Median score',
# 'Random agent Std',
# 'RL agent Mean score',
# 'RL agent Median score',
# 'RL agent Std']
# writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
#
# if not file_exists:
# print("File does not exist")
# writer.writeheader()
#
# writer.writerow({
# 'Grid Size': grid_size,
# 'Number of Colors': color,
# 'statistics': '%f' % stat,
# 'p-value': '%f' % p,
# 'Top agent Mean score': round(statistics.mean(df_1), 2),
# 'Top agent Median score': round(statistics.median(df_1), 2),
# 'Top agent Std': round(statistics.stdev(df_1), 2),
# 'Bottom agent Mean score': round(statistics.mean(df_2), 2),
# 'Bottom agent Median score': round(statistics.median(df_2), 2),
# 'Bottom agent Std': round(statistics.stdev(df_2), 2),
# 'Random agent Mean score': round(statistics.mean(df_3), 2),
# 'Random agent Median score': round(statistics.median(df_3), 2),
# 'Random agent Std': round(statistics.stdev(df_3), 2),
# 'RL agent Mean score': round(statistics.mean(df_4), 2),
# 'RL agent Median score': round(statistics.median(df_4), 2),
# 'RL agent Std': round(statistics.stdev(df_4), 2)
# })
#
# print('Statistics=%f, p=%f' % (stat, p))
# interpret
alpha = 0.05
if p > alpha:
print(f'Same distribution (fail to reject H0) for {grid_size} and {color}')
else:
print(f'Different distribution (reject H0) for {grid_size} and {color}')
#combine three groups into one array
data = np.array([df_1, df_3, df_2, df_4])
#perform Nemenyi post-hoc test
print(sp.posthoc_nemenyi_friedman(data.T).round(3))
......@@ -2,6 +2,7 @@ import pandas as pd
import matplotlib.pyplot as plot
data = pd.read_csv("agents_eval.csv")
color_list = ["tab:blue", "tab:orange", "tab:green", "tab:red"]
top_scores = data[data['Agent'] == 'top_agent']['Score Mean'].round(2).tolist()
random_scores = data[data['Agent'] == 'random_agent']['Score Mean'].round(2).tolist()
......@@ -18,17 +19,35 @@ df = pd.DataFrame({'RBA1': top_scores,
'RBA2': bottom_scores,
'RL agent': rl_scores}, index=index)
ax = df.plot.bar(rot=0, width=0.25)
ax.set_ylabel("Mean score")
# ax = df.plot.bar(rot=0, width=0.5)
ax = df.plot(kind="bar", rot=0, width=0.5, yerr=[data[data['Agent'] == 'top_agent']['Score Std'],
data[data['Agent'] == 'random_agent']['Score Std'],
data[data['Agent'] == 'bottom_agent']['Score Std'],
data[data['Agent'] == 'RL Agent(PPO)']['Score Std']],
color=color_list, capsize=2, error_kw=dict(ecolor='darkslategray'))
ax.yaxis.grid()
ax.set_axisbelow(True)
ax.figure.set_size_inches(8, 5)
# ax.figure.update_layout(barmode='group', bargap=0.15,)
ax.set_ylabel("Score")
plot.show()
plot.close()
# ====================================
df = pd.DataFrame({'RBA1': top_avalanche_score,
'random agent': random_avalanche_score,
'RBA2': bottom_avalanche_score,
'RL agent': rl_avalanche_score}, index=index)
ax = df.plot.bar(rot=0, width=0.25)
ax.set_ylabel("Mean non-deterministic score")
# ax = df.plot.bar(rot=0, width=0.5)
ax = df.plot(kind="bar", rot=0, width=0.5, yerr=[data[data['Agent'] == 'top_agent']['Avalanche score Std'],
data[data['Agent'] == 'random_agent']['Avalanche score Std'],
data[data['Agent'] == 'bottom_agent']['Avalanche score Std'],
data[data['Agent'] == 'RL Agent(PPO)']['Avalanche score Std']],
color=color_list, capsize=2, error_kw=dict(ecolor='darkslategray'))
ax.yaxis.grid()
ax.set_axisbelow(True)
ax.figure.set_size_inches(8, 5)
ax.set_ylabel("Non-deterministic score")
plot.show()
plot.close()
# ====================================
......@@ -37,8 +56,14 @@ df = pd.DataFrame({'RBA1': data[data['Agent'] == 'top_agent']['Deterministic sco
'RBA2': data[data['Agent'] == 'bottom_agent']['Deterministic score Mean'].tolist(),
'RL agent': data[data['Agent'] == 'RL Agent(PPO)']['Deterministic score Mean'].tolist()}, index=index)
ax = df.plot.bar(rot=0, width=0.25)
ax = df.plot(kind="bar", rot=0, width=0.5, yerr=[data[data['Agent'] == 'top_agent']['Deterministic score Std'],
data[data['Agent'] == 'random_agent']['Deterministic score Std'],
data[data['Agent'] == 'bottom_agent']['Deterministic score Std'],
data[data['Agent'] == 'RL Agent(PPO)']['Deterministic score Std']],
color=color_list, capsize=2, error_kw=dict(ecolor='darkslategray'))
ax.yaxis.grid()
ax.set_axisbelow(True)
ax.figure.set_size_inches(8, 5)
plot.legend(loc='lower left')
ax.set_ylabel("Mean deterministic score")
ax.set_ylabel("Deterministic score")
plot.show()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment