Commit a4aaeccb authored by Melanie Bruns's avatar Melanie Bruns
Browse files

Merge branch 'feature-#731-averaged-results' into 'master'

Add script to average evaluation results

See merge request !165
parents d7640725 cb5fc4be
Pipeline #17261 passed with stages
in 4 minutes and 37 seconds
import matplotlib.pyplot as plt
import sys
import os
import privateMessageAnalysis
import delayDistributionAnalysis
import broadcastAnalysis
import multicastAnalysis
import readFileUtilities
# Script responsible for averaging multiple runs and outputting graphics for average delivery rates and delay
# distribution for all message types.
#
# Assumes perl scripts
# * messageDelayAnalyzer.pl,
# * broadcastMessageAnalyzer.pl and
# * multicastMessageAnalyzer.pl
# have already been called and their results stored in messageDelayAnalysis.txt, broadcastMessageAnalysis.txt and
# multicastMessageAnalysis.txt respectively.
# Also assumes that realisticScenario_DeliveryProbabilityReport.txt exists.
#
# An easy way to call these perl scripts is by running reportSummary.py.
def averageLists(tuples):
"""Averages a list of tuples. Each tuple is expected to consist of multiple lists that all have the same length.
Example:
tuples: [
([3, 6, 9], [4, 2, 8], [5, 1, 1]),
([3, 6, 9], [3, 1, 1], [7, 8, 10])]
result: ([3, 6, 9],[3.5, 1.5, 4.5],[6, 4.5, 5.5])
"""
# Create averages for each element in the 'matrix' created by tuples of lists.
# Those averages are indexed by first the tuple index and then the list index.
# First, sum up everything.
sums = {}
for singleTuple in tuples:
for (tupleElementIndex, list) in enumerate(singleTuple):
if not tupleElementIndex in sums:
sums[tupleElementIndex] = {}
for (listIdx, listElement) in enumerate(list):
if not listIdx in sums[tupleElementIndex]:
sums[tupleElementIndex][listIdx] = listElement
else:
sums[tupleElementIndex][listIdx] += listElement
# Then, divide everything.
averagedLists = []
for tupleElementIndex, tupleAverages in sums.items():
averagedList = []
for listIdx, value in tupleAverages.items():
averagedList.append(sums[tupleElementIndex][listIdx] / len(tuples))
averagedLists.append(averagedList)
return tuple(averagedLists)
def average(tuples):
"""Averages a list of tuples. Each tuple is expected to consist of numeric values.
Example:
tuples: [(3, 4, 5), (3, 3, 7)]
result: (3, 3.5, 6])
"""
# Create averages for each element in the tuples.
# Those averages are indexed by the tuple index.
# First, sum up everything.
sums = {}
for singleTuple in tuples:
for (tupleElementIndex, element) in enumerate(singleTuple):
if not tupleElementIndex in sums:
sums[tupleElementIndex] = element
else:
sums[tupleElementIndex] += element
# Then, divide everything.
averaged = []
for idx, value in sums.items():
averaged.append(value / len(tuples))
return tuple(averaged)
def unifyTimePoints(tuples):
"""Unifies the first list among a list of tuples of lists.
Each tuple is expected to have a list as first element. The longest of those lists is copied over to all other tuples.
Example:
tuples: [
([3, 6, 9], [4, 2, 8], [5, 1, 1]),
([3, 6, 9, 12, 15], [3, 1, 1, 4, 6], [7, 8, 10, 23, 45]),
([3], [1], [70])]
result: [
([3, 6, 9, 12, 15], [4, 2, 8], [5, 1, 1]),
([3, 6, 9, 12, 15], [3, 1, 1, 4, 6], [7, 8, 10, 23, 45]),
([3, 6, 9, 12, 15], [1], [70])]
"""
longestFirstList = []
for singleTuple in tuples:
if len(singleTuple[0]) > len(longestFirstList):
longestFirstList = singleTuple[0]
tuplesWithUnifiedTimePoints = []
for singleTuple in tuples:
tupleAsList = list(singleTuple)
tupleAsList[0] = longestFirstList
tuplesWithUnifiedTimePoints.append(tuple(tupleAsList))
return tuplesWithUnifiedTimePoints
def padByZeros(tuples, tupleIndices):
"""Takes in a list of tuples of lists. Makes sure all tuple elements matching the provided indices have the same
length as the first tuple element by padding them by zeros.
Example:
tuples: [
([3, 6, 9, 12, 15], [4, 2, 8], [5, 1, 1]),
([3, 6, 9, 12, 15], [3, 1, 1, 4, 6], [7, 8, 10, 23, 45]),
([3, 6, 9, 12, 15], [1], [70])]
tupleIndices: [1]
tuples after function call: [
([3, 6, 9, 12, 15], [4, 2, 8, 0, 0], [5, 1, 1]),
([3, 6, 9, 12, 15], [3, 1, 1, 4, 6], [7, 8, 10, 23, 45]),
([3, 6, 9, 12, 15], [1, 0, 0, 0, 0], [70])]
"""
for singleTuple in tuples:
for tupleIdx in tupleIndices:
list = singleTuple[tupleIdx]
while len(list) < len(singleTuple[0]):
list.append(0)
def padByReplication(tuples, tupleIndices):
"""
Takes in a list of tuples of lists. Makes sure all tuple elements matching the provided indices have the same
length as the first tuple element by padding them with their latest element.
Example:
tuples: [
([3, 6, 9, 12, 15], [4, 2, 8, 0, 0], [5, 1, 1]),
([3, 6, 9, 12, 15], [3, 1, 1, 4, 6], [7, 8, 10, 23, 45]),
([3, 6, 9, 12, 15], [1, 0, 0, 0, 0], [70])]
tupleIndices: [2]
tuples after function call: [
([3, 6, 9, 12, 15], [4, 2, 8, 0, 0], [5, 1, 1, 1, 1]),
([3, 6, 9, 12, 15], [3, 1, 1, 4, 6], [7, 8, 10, 23, 45]),
([3, 6, 9, 12, 15], [1, 0, 0, 0, 0], [70, 70, 70, 70, 70])]
"""
for singleTuple in tuples:
for tupleIdx in tupleIndices:
list = singleTuple[tupleIdx]
lastElement = list[len(list) - 1]
while len(list) < len(singleTuple[0]):
list.append(lastElement)
def averageFiles(reportsDirectory, seeds):
"""
Parses all necessary files, adds additional information where needed, and returns (in that order) averaged
one-to-one delivery rates, an averaged one-to-one delay distribution, averaged multicast delivery rates,
an averaged multicast delay distribution, averaged broadcast delivery numbers, and an averaged broadcast delay
distribution.
:param reportsDirectory: Path to directory containing folders for different runs, where each folder contains all relevant reports /
analysis files
:param seeds: List of folder names containing relevant reports / analysis files, e.g. '1 2 4 5'.
:return: averaged one-to-one delivery rates, averaged one-to-one delay distribution,
averaged multicast delivery rates, an averaged multicast delay distribution,
averaged broadcast delivery numbers, and an averaged broadcast delay distribution.
"""
# Read all information from files
oneToOneDeliveryRates = []
oneToOneDelayDistributions = []
multicastDeliveryRates = []
multicastDelayDistributions = []
broadcastDeliveryNumbers = []
broadcastDelayDistributions = []
for seed in seeds.split():
directory = os.path.join(reportsDirectory, seed)
delayAnalysis = os.path.join(directory, "messageDelayAnalysis.txt")
oneToOneDeliveryRates.append(
privateMessageAnalysis.parseDeliveryProbabilityReport(os.path.join(directory, "realisticScenario_DeliveryProbabilityReport.txt")))
oneToOneDelayDistributions.append(
delayDistributionAnalysis.parseDelayAnalysis(delayAnalysis, messageType="ONE_TO_ONE", messagePrio=0))
multicastDeliveryRates.append(
multicastAnalysis.parseMulticastAnalysis(os.path.join(directory, "multicastMessageAnalysis.txt")))
multicastDelayDistributions.append(
delayDistributionAnalysis.parseDelayAnalysis(delayAnalysis, messageType="MULTICAST", messagePrio=1))
broadcastDeliveryNumbers.append(
broadcastAnalysis.parseBroadcastAnalysis(os.path.join(directory, "broadcastMessageAnalysis.txt"), priority=5))
broadcastDelayDistributions.append(
delayDistributionAnalysis.parseDelayAnalysis(delayAnalysis, messageType="BROADCAST", messagePrio=5))
# Make sure all time points exist for all seeds
oneToOneDelayDistributions = unifyTimePoints(oneToOneDelayDistributions)
multicastDelayDistributions = unifyTimePoints(multicastDelayDistributions)
broadcastDeliveryNumbers = unifyTimePoints(broadcastDeliveryNumbers)
broadcastDelayDistributions = unifyTimePoints(broadcastDelayDistributions)
# Add missing values for delay distributions.
padByZeros(oneToOneDelayDistributions, [1])
padByZeros(multicastDelayDistributions, [1])
padByZeros(broadcastDelayDistributions, [1])
padByReplication(oneToOneDelayDistributions, [2])
padByReplication(multicastDelayDistributions, [2])
padByReplication(broadcastDelayDistributions, [2])
# Add missing values for broadcast delivery numbers.
padByReplication(broadcastDeliveryNumbers, [1, 2])
# Return everything as averaged.
return average(oneToOneDeliveryRates),\
averageLists(oneToOneDelayDistributions), \
averageLists(multicastDeliveryRates),\
averageLists(multicastDelayDistributions),\
averageLists(broadcastDeliveryNumbers),\
averageLists(broadcastDelayDistributions)
def main(reportsDirectory, seeds):
"""Main function call to create averaged graphics.
Parameters
----------
reportsDirectory
Path to directory containing folders for different runs, where each folder contains all relevant reports /
analysis files
seeds
List of folder names containing relevant reports / analysis files, e.g. '1 2 4 5'.
"""
# Parse and average all information.
averagedOneToOneDeliveryRates, averagedOneToOneDelayDistribution,\
averagedMulticastDeliveryRates, averagedMulticastDelayDistribution,\
averagedBroadcastDeliveryNumbers, averagedBroadcastDelayDistribution = averageFiles(reportsDirectory, seeds)
# Use it to create graphics.
graphicsDirectory = os.path.join(reportsDirectory, "graphics/")
if not os.path.exists(graphicsDirectory):
os.makedirs(graphicsDirectory)
delayDistributionAnalysis.createDelayGraphicInFile(
*averagedOneToOneDelayDistribution,
title='Delay distribution of delivered ONE_TO_ONE messages',
fileName=readFileUtilities.getAbsoluteOneToOneMessageDelayPath(graphicsDirectory))
delayDistributionAnalysis.createDelayGraphicInFile(
*averagedMulticastDelayDistribution,
title='Delay distribution of delivered MULTICAST messages',
fileName=readFileUtilities.getAbsoluteMulticastDelayPath(graphicsDirectory))
delayDistributionAnalysis.createDelayGraphicInFile(
*averagedBroadcastDelayDistribution,
title='Delay distribution of delivered BROADCAST messages',
fileName=readFileUtilities.getAbsoluteBroadcastDelayPath(graphicsDirectory, prio=5))
privateMessageAnalysis.createDeliveryPieChart(*averagedOneToOneDeliveryRates)
plt.savefig(readFileUtilities.getAbsoluteDeliveryRatePath(graphicsDirectory))
plt.close()
multicastAnalysis.drawPlots(*averagedMulticastDeliveryRates)
plt.savefig(readFileUtilities.getAbsoluteMulticastAnalysisPath(graphicsDirectory))
plt.close()
broadcastAnalysis.drawPlots(*averagedBroadcastDeliveryNumbers, priority=5)
plt.savefig(readFileUtilities.getAbsoluteBroadcastAnalysisPath(graphicsDirectory))
plt.close()
# Make sure script can be called from command line.
if __name__ == "__main__":
main(sys.argv[1], sys.argv[2])
\ No newline at end of file
......@@ -42,12 +42,12 @@ def parseDeliveryProbabilityReport(fileName):
def createDeliveryPieChart(created, delivered, delivery_prob):
"""Creates a graphical presentation of delivery probability."""
values=[delivery_prob, 1-delivery_prob]
labels=["delivered:\n{p:.1f}% ({t})".format(p=delivery_prob*100, t=delivered),
"not delivered:\n{p:.1f}% ({t})".format(p=(1-delivery_prob)*100, t=(created-delivered))]
labels=["delivered:\n{p:.1f}% ({t:.1f})".format(p=delivery_prob*100, t=delivered),
"not delivered:\n{p:.1f}% ({t:.1f})".format(p=(1-delivery_prob)*100, t=(created-delivered))]
# Create pie chart.
fig1, ax1 = plt.subplots()
_, texts = ax1.pie(values, labels=labels, shadow=True, explode=(0.1, 0), labeldistance=0.3)
_, texts = ax1.pie(values, labels=labels, shadow=True, explode=(0.1, 0), labeldistance=0.2)
for text in texts:
text.set_fontsize(12)
ax1.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment