[haizea-commit] r832 - in branches/1.1/src/haizea: cli lwf
haizea-commit at mailman.cs.uchicago.edu
haizea-commit at mailman.cs.uchicago.edu
Wed Jul 21 13:27:14 CDT 2010
Author: borja
Date: 2010-07-21 13:27:13 -0500 (Wed, 21 Jul 2010)
New Revision: 832
Modified:
branches/1.1/src/haizea/cli/commands.py
branches/1.1/src/haizea/lwf/analysis.py
branches/1.1/src/haizea/lwf/generators.py
Log:
haizea-lwf-stats now prints more stats
Modified: branches/1.1/src/haizea/cli/commands.py
===================================================================
--- branches/1.1/src/haizea/cli/commands.py 2010-07-16 16:44:33 UTC (rev 831)
+++ branches/1.1/src/haizea/cli/commands.py 2010-07-21 18:27:13 UTC (rev 832)
@@ -21,7 +21,7 @@
from haizea.core.configfile import HaizeaConfig, HaizeaMultiConfig
from haizea.core.accounting import AccountingDataCollection
from haizea.common.config import ConfigException
-from haizea.common.stats import percentile
+from haizea.common.stats import percentile, print_percentiles
from haizea.cli.optionparser import Option
from haizea.cli import Command
from haizea.lwf.generators import LWFGenerator, LWFAnnotationGenerator
@@ -407,6 +407,10 @@
help = """
Annotation file
"""))
+ self.optparser.add_option(Option("-v", "--verbose", action="store_true", dest="verbose",
+ help = """
+ Verbose
+ """))
self.optparser.add_option(Option("-l", "--utilization-length", action="store", type="string", dest="utilization_length",
help = """
Length of the utilization interval in format DD:HH:MM:SS. Default is until
@@ -422,7 +426,7 @@
if utilization_length != None:
utilization_length = Parser.DateTimeDeltaFromString(utilization_length)
- analyser = LWFAnalyser(infile, utilization_length, annotationfile)
+ analyser = LWFAnalyser(infile, utilization_length, annotationfile, self.opt.verbose)
analyser.analyse()
@@ -884,21 +888,15 @@
outfile.close()
slowdowns.sort()
-
total_capacity = site_num_nodes * (to_time - from_time).seconds
+ print utilization, total_capacity
utilization = float(utilization) / float(total_capacity)
utilization_no_ramp = float(utilization_no_ramp) / float(total_capacity)
-
+
if len(slowdowns) > 0:
print "SLOWDOWNS"
print "---------"
- print "min: %.2f" % slowdowns[0]
- print "10p: %.2f" % percentile(slowdowns, 0.1)
- print "25p: %.2f" % percentile(slowdowns, 0.25)
- print "med: %.2f" % percentile(slowdowns, 0.5)
- print "75p: %.2f" % percentile(slowdowns, 0.75)
- print "90p: %.2f" % percentile(slowdowns, 0.9)
- print "max: %.2f" % slowdowns[-1]
+ print_percentiles(slowdowns)
print
print "USERS"
print "-----"
Modified: branches/1.1/src/haizea/lwf/analysis.py
===================================================================
--- branches/1.1/src/haizea/lwf/analysis.py 2010-07-16 16:44:33 UTC (rev 831)
+++ branches/1.1/src/haizea/lwf/analysis.py 2010-07-21 18:27:13 UTC (rev 832)
@@ -19,12 +19,12 @@
from haizea.core.leases import LeaseWorkload, Site, UnmanagedSoftwareEnvironment,\
DiskImageSoftwareEnvironment, LeaseAnnotations
import operator
+from haizea.common.stats import percentile, print_percentiles, print_distribution
-
class LWFAnalyser(object):
- def __init__(self, lwffile, utilization_length, annotationfile):
+ def __init__(self, lwffile, utilization_length, annotationfile, verbose = False):
# Arbitrary start time
self.starttime = DateTime(2006,11,25,13)
@@ -39,19 +39,47 @@
if annotationfile != None:
annotations = LeaseAnnotations.from_xml_file(annotationfile)
annotations.apply_to_leases(self.workload.get_leases())
+
+ self.verbose = verbose
def analyse(self):
- utilization = 0
+ requtilization = 0
+ actutilization = 0
software = {"Unmanaged": 0}
+ nnodes = []
+ reqdurations = []
+ actdurations = []
nleases = len(self.workload.get_leases())
for lease in self.workload.get_leases():
- if lease.start.requested + lease.duration.requested > self.starttime + self.utilization_length:
- duration = (self.starttime + self.utilization_length - lease.start.requested).seconds
+ if lease.start.requested == "Unspecified":
+ start = lease.submit_time
+ else:
+ start = lease.start.requested
+
+ if start + lease.duration.requested > self.starttime + self.utilization_length:
+ reqduration = (self.starttime + self.utilization_length - start).seconds
else:
- duration = lease.duration.requested.seconds
+ reqduration = lease.duration.requested.seconds
+
+ if lease.duration.known != None:
+ if start + lease.duration.known > self.starttime + self.utilization_length:
+ actduration = (self.starttime + self.utilization_length - start).seconds
+ else:
+ actduration = lease.duration.known.seconds
+ else:
+ actduration = reqduration
+
for res in lease.requested_resources.values():
for i in range(1,res.get_ninstances("CPU") + 1):
- utilization += (res.get_quantity_instance("CPU", i) / 100.0) * duration
+ requtilization += (res.get_quantity_instance("CPU", i) / 100.0) * reqduration
+ actutilization += (res.get_quantity_instance("CPU", i) / 100.0) * actduration
+
+ nnodes.append(len(lease.requested_resources))
+ reqdurations.append(lease.duration.requested.seconds)
+
+ if lease.duration.known != None:
+ actdurations.append(lease.duration.known.seconds)
+
if isinstance(lease.software, UnmanagedSoftwareEnvironment):
software["Unmanaged"] += 1
elif isinstance(lease.software, DiskImageSoftwareEnvironment):
@@ -64,10 +92,47 @@
for res in self.site.nodes.get_all_nodes().values():
for i in range(1,res.get_ninstances("CPU") + 1):
max_utilization += (res.get_quantity_instance("CPU", i)/100.0) * duration
-
- print "Utilization: %.2f%%" % ((utilization / max_utilization) * 100.0)
+
+ if self.verbose:
+ reqdurd = {}
+ nnodesd = {}
+ for reqduration in reqdurations:
+ reqdurd[reqduration] = reqdurd.setdefault(reqduration, 0) +1
+ for n in nnodes:
+ nnodesd[n] = nnodesd.setdefault(n, 0) +1
+
+
+ print actutilization
+ print max_utilization
+ print "Requested utilization: %.2f%%" % ((requtilization / max_utilization) * 100.0)
+ print " Actual utilization: %.2f%%" % ((actutilization / max_utilization) * 100.0)
print
- sorted_images = sorted(software.iteritems(), key=operator.itemgetter(1), reverse=True)
- for image, count in sorted_images:
- print "%s: %i (%.2f%%)" % (image, count, (float(count)/nleases)*100)
-
\ No newline at end of file
+ #sorted_images = sorted(software.iteritems(), key=operator.itemgetter(1), reverse=True)
+ print "NODES"
+ print "-----"
+ print_percentiles(nnodes)
+ print
+ if self.verbose:
+ print "NODES (distribution)"
+ print "--------------------"
+ print_distribution(nnodesd, nleases)
+ print
+ print "REQUESTED DURATIONS"
+ print "-------------------"
+ print_percentiles(reqdurations)
+ print
+ if self.verbose:
+ print "REQUESTED DURATIONS (distribution)"
+ print "----------------------------------"
+ print_distribution(reqdurd, nleases)
+ print
+ print "ACTUAL DURATIONS"
+ print "----------------"
+ print_percentiles(actdurations)
+ print
+ print "IMAGES"
+ print "------"
+ print_distribution(software, nleases)
+ #for image, count in sorted_images:
+ # print "%s: %i (%.2f%%)" % (image, count, (float(count)/nleases)*100)
+ print
\ No newline at end of file
Modified: branches/1.1/src/haizea/lwf/generators.py
===================================================================
--- branches/1.1/src/haizea/lwf/generators.py 2010-07-16 16:44:33 UTC (rev 831)
+++ branches/1.1/src/haizea/lwf/generators.py 2010-07-21 18:27:13 UTC (rev 832)
@@ -92,6 +92,7 @@
self.numnodes_dist = self._get_dist(FileGenerator.NODES_SEC)
self.software_dist = self._get_dist(FileGenerator.SOFTWARE_SEC)
+ self.user_rates = {}
def _get_start(self, type, lease = None):
@@ -426,7 +427,7 @@
lease_workload = LeaseWorkload.from_xml_file(self.lwffile)
leases = lease_workload.get_leases()
for lease in leases:
- annotations = self.__gen_annotation(lease)
+ annotation = self.__gen_annotation(lease)
annotations[lease.id] = annotation
attributes = self._get_attributes()
More information about the Haizea-commit
mailing list