[haizea-commit] r799 - in branches/1.1/src/haizea: cli common core lwf
haizea-commit at mailman.cs.uchicago.edu
haizea-commit at mailman.cs.uchicago.edu
Wed Jun 2 12:30:03 CDT 2010
Author: borja
Date: 2010-06-02 12:30:02 -0500 (Wed, 02 Jun 2010)
New Revision: 799
Modified:
branches/1.1/src/haizea/cli/commands.py
branches/1.1/src/haizea/common/stats.py
branches/1.1/src/haizea/core/leases.py
branches/1.1/src/haizea/lwf/analysis.py
branches/1.1/src/haizea/lwf/generators.py
Log:
- Added support for lease annotation files where the annotations are not bound to a specific lease
- haizea-lwf-stats now prints a summary of requested images
- Tweaked discrete distributions in stats module
Modified: branches/1.1/src/haizea/cli/commands.py
===================================================================
--- branches/1.1/src/haizea/cli/commands.py 2010-06-01 18:33:29 UTC (rev 798)
+++ branches/1.1/src/haizea/cli/commands.py 2010-06-02 17:30:02 UTC (rev 799)
@@ -390,6 +390,10 @@
help = """
Input file
"""))
+ self.optparser.add_option(Option("-a", "--annotation", action="store", type="string", dest="annotationf",
+ help = """
+ Annotation file
+ """))
self.optparser.add_option(Option("-l", "--utilization-length", action="store", type="string", dest="utilization_length",
help = """
Length of the utilization interval in format DD:HH:MM:SS. Default is until
@@ -400,11 +404,12 @@
self.parse_options()
infile = self.opt.inf
+ annotationfile = self.opt.annotationf
utilization_length = self.opt.utilization_length
if utilization_length != None:
utilization_length = Parser.DateTimeDeltaFromString(utilization_length)
- analyser = LWFAnalyser(infile, utilization_length)
+ analyser = LWFAnalyser(infile, utilization_length, annotationfile)
analyser.analyse()
@@ -414,10 +419,14 @@
def __init__(self, argv):
Command.__init__(self, argv)
- self.optparser.add_option(Option("-i", "--in", action="store", type="string", dest="inf", required=True,
+ self.optparser.add_option(Option("-i", "--in", action="store", type="string", dest="inf",
help = """
LWF file
"""))
+ self.optparser.add_option(Option("-n", "--num-annotations", action="store", type="int", dest="nleases",
+ help = """
+ Number of annotations
+ """))
self.optparser.add_option(Option("-o", "--out", action="store", type="string", dest="outf", required=True,
help = """
Annotation file
@@ -431,10 +440,11 @@
self.parse_options()
infile = self.opt.inf
+ nleases = self.opt.nleases
outfile = self.opt.outf
conffile = self.opt.conf
- generator = LWFAnnotationGenerator(infile, outfile, conffile)
+ generator = LWFAnnotationGenerator(infile, nleases, outfile, conffile)
generator.generate()
Modified: branches/1.1/src/haizea/common/stats.py
===================================================================
--- branches/1.1/src/haizea/common/stats.py 2010-06-01 18:33:29 UTC (rev 798)
+++ branches/1.1/src/haizea/common/stats.py 2010-06-02 17:30:02 UTC (rev 799)
@@ -126,23 +126,33 @@
def __init__(self, values):
self.values = values
self.num_values = len(self.values)
+ self.__distribution = None
+
+ def _set_distribution(self, dis):
+ self.__distribution = dis
# Expects value in [0,1)
def _get_from_prob(self, prob):
pos = int(math.floor(prob * self.num_values))
return self.values[pos]
-
-class DiscreteUniformDistribution(DiscreteDistribution):
- def __init__(self, values):
- DiscreteDistribution.__init__(self, values)
- self.__distribution = UniformDistribution(0,1)
-
+
def seed(self, x):
self.__distribution.seed(x)
def get(self):
return self._get_from_prob(self.__distribution.get())
+
+class DiscreteUniformDistribution(DiscreteDistribution):
+ def __init__(self, values):
+ DiscreteDistribution.__init__(self, values)
+ self._set_distribution(UniformDistribution(0,1))
+
+class DiscreteTruncatedParetoDistribution(DiscreteDistribution):
+ def __init__(self, values, scale, alpha, invert = False):
+ DiscreteDistribution.__init__(self, values)
+ self._set_distribution(TruncatedParetoDistribution(0,1,scale,alpha,invert))
+
def percentile(values, percent):
pos = int(len(values) * percent)
return values[pos]
Modified: branches/1.1/src/haizea/core/leases.py
===================================================================
--- branches/1.1/src/haizea/core/leases.py 2010-06-01 18:33:29 UTC (rev 798)
+++ branches/1.1/src/haizea/core/leases.py 2010-06-02 17:30:02 UTC (rev 799)
@@ -1369,16 +1369,18 @@
"""
annotation = ET.Element("lease-annotation")
- annotation.set("id", str(self.lease_id))
+ if self.lease_id != None:
+ annotation.set("id", str(self.lease_id))
- start = ET.SubElement(annotation, "start")
- if self.start.requested == Timestamp.UNSPECIFIED:
- pass # empty start element
- elif self.start.requested == Timestamp.NOW:
- ET.SubElement(start, "now") #empty now element
- else:
- exact = ET.SubElement(start, "exact")
- exact.set("time", "+" + str(self.start.requested))
+ if self.start != None:
+ start = ET.SubElement(annotation, "start")
+ if self.start.requested == Timestamp.UNSPECIFIED:
+ pass # empty start element
+ elif self.start.requested == Timestamp.NOW:
+ ET.SubElement(start, "now") #empty now element
+ else:
+ exact = ET.SubElement(start, "exact")
+ exact.set("time", "+" + str(self.start.requested))
if self.deadline != None:
deadline = ET.SubElement(annotation, "deadline")
@@ -1420,17 +1422,15 @@
Arguments:
annotations -- A dictionary of annotations
- """
+ """
+ if isinstance(annotations, list):
+ self.lease_specific_annotations = False
+ elif isinstance(annotations, dict):
+ self.lease_specific_annotations = True
self.annotations = annotations
self.attributes = attributes
- def apply_to_leases(self, leases):
- """Apply annotations to a workload
-
- """
- for lease in [l for l in leases if self.has_annotation(l.id)]:
- annotation = self.get_annotation(lease.id)
-
+ def __apply_to_lease(self, lease, annotation):
if annotation.start != None:
if annotation.start.requested in (Timestamp.NOW, Timestamp.UNSPECIFIED):
lease.start.requested = annotation.start.requested
@@ -1445,18 +1445,18 @@
if annotation.extras != None:
lease.extras.update(annotation.extras)
-
- def get_annotation(self, lease_id):
- """...
+
+ def apply_to_leases(self, leases):
+ """Apply annotations to a workload
- """
- return self.annotations[lease_id]
-
- def has_annotation(self, lease_id):
- """...
-
- """
- return self.annotations.has_key(lease_id)
+ """
+ if self.lease_specific_annotations:
+ for lease in [l for l in leases if self.annotations.has_key(l.id)]:
+ annotation = self.annotations[lease.id]
+ self.__apply_to_lease(lease, annotation)
+ else:
+ for lease, annotation in zip(leases, self.annotations):
+ self.__apply_to_lease(lease, annotation)
@classmethod
def from_xml_file(cls, xml_file):
@@ -1479,16 +1479,27 @@
element -- Element object containing a "<lease-annotations>" element.
"""
annotation_elems = element.findall("lease-annotation")
- annotations = {}
+ annotations_dict = {}
+ annotations_list = []
for annotation_elem in annotation_elems:
- lease_id = int(annotation_elem.get("id"))
- annotations[lease_id] = LeaseAnnotation.from_xml_element(annotation_elem)
+ annotation = LeaseAnnotation.from_xml_element(annotation_elem)
+ if annotation.lease_id == None:
+ annotations_list.append(annotation)
+ else:
+ annotations_dict[annotation.lease_id] = annotation
attributes = {}
attributes_elem = element.find("attributes")
if attributes_elem != None:
for attr_elem in attributes_elem:
attributes[attr_elem.get("name")] = attr_elem.get("value")
+
+ if len(annotations_list) != 0 and len(annotations_dict) != 0:
+ raise Exception #TODO: raise something more meaningful
+ elif len(annotations_list) == 0:
+ annotations = annotations_dict
+ elif len(annotations_dict) == 0:
+ annotations = annotations_list
return cls(annotations, attributes)
@@ -1499,18 +1510,23 @@
lease XML format.
"""
- annotations = ET.Element("lease-annotations")
+ annotations_elem = ET.Element("lease-annotations")
- attributes = ET.SubElement(annotations, "attributes")
+ attributes = ET.SubElement(annotations_elem, "attributes")
for name, value in self.attributes.items():
attr_elem = ET.SubElement(attributes, "attr")
attr_elem.set("name", name)
attr_elem.set("value", value)
- for annotation in self.annotations.values():
- annotations.append(annotation.to_xml())
+ if self.lease_specific_annotations:
+ annotations = self.annotations.values()
+ else:
+ annotations = self.annotations
+
+ for annotation in annotations:
+ annotations_elem.append(annotation.to_xml())
- return annotations
+ return annotations_elem
def to_xml_string(self):
"""Returns a string XML representation of the lease
Modified: branches/1.1/src/haizea/lwf/analysis.py
===================================================================
--- branches/1.1/src/haizea/lwf/analysis.py 2010-06-01 18:33:29 UTC (rev 798)
+++ branches/1.1/src/haizea/lwf/analysis.py 2010-06-02 17:30:02 UTC (rev 799)
@@ -16,13 +16,15 @@
# limitations under the License. #
# -------------------------------------------------------------------------- #
from mx.DateTime import DateTime
-from haizea.core.leases import LeaseWorkload, Site
+from haizea.core.leases import LeaseWorkload, Site, UnmanagedSoftwareEnvironment,\
+ DiskImageSoftwareEnvironment, LeaseAnnotations
+import operator
class LWFAnalyser(object):
- def __init__(self, lwffile, utilization_length):
+ def __init__(self, lwffile, utilization_length, annotationfile):
# Arbitrary start time
self.starttime = DateTime(2006,11,25,13)
@@ -33,10 +35,15 @@
self.utilization_length = self.workload.get_leases()[-1].submit_time - self.starttime
else:
self.utilization_length = utilization_length
- print self.utilization_length
+
+ if annotationfile != None:
+ annotations = LeaseAnnotations.from_xml_file(annotationfile)
+ annotations.apply_to_leases(self.workload.get_leases())
def analyse(self):
utilization = 0
+ software = {"Unmanaged": 0}
+ nleases = len(self.workload.get_leases())
for lease in self.workload.get_leases():
if lease.start.requested + lease.duration.requested > self.starttime + self.utilization_length:
duration = self.starttime + self.utilization_length - lease.start.requested
@@ -45,7 +52,12 @@
for res in lease.requested_resources.values():
for i in range(1,res.get_ninstances("CPU") + 1):
utilization += res.get_quantity_instance("CPU", i) * duration
-
+ if isinstance(lease.software, UnmanagedSoftwareEnvironment):
+ software["Unmanaged"] += 1
+ elif isinstance(lease.software, DiskImageSoftwareEnvironment):
+ image = lease.software.image_id
+ software[image] = software.setdefault(image, 0) +1
+
if self.site != None:
max_utilization = 0
duration = self.utilization_length.seconds
@@ -55,4 +67,8 @@
print "Utilization: %.2f%%" % (utilization / max_utilization)
+ print
+ sorted_images = sorted(software.iteritems(), key=operator.itemgetter(1), reverse=True)
+ for image, count in sorted_images:
+ print "%s: %i (%.2f%%)" % (image, count, (float(count)/nleases)*100)
\ No newline at end of file
Modified: branches/1.1/src/haizea/lwf/generators.py
===================================================================
--- branches/1.1/src/haizea/lwf/generators.py 2010-06-01 18:33:29 UTC (rev 798)
+++ branches/1.1/src/haizea/lwf/generators.py 2010-06-02 17:30:02 UTC (rev 799)
@@ -33,7 +33,8 @@
class FileGenerator(object):
GENERAL_SEC = "general"
-
+ ATTRIBUTES_OPT = "attributes"
+
TYPE_OPT = "type"
DISTRIBUTION_OPT = "distribution"
DISCRETE_OPT = "discrete"
@@ -75,14 +76,22 @@
self.config.readfp(conffile)
self.startdelay_dist = self._get_dist(FileGenerator.START_DELAY_SEC)
+ if self.startdelay_dist != None:
+ self.start_type = self.config.get(FileGenerator.START_DELAY_SEC, FileGenerator.TYPE_OPT)
+ else:
+ self.start_type = None
+
self.deadlinestretch_dist = self._get_dist(FileGenerator.DEADLINE_STRETCH_SEC)
+ if self.deadlinestretch_dist != None:
+ self.deadline_type = self.config.get(FileGenerator.DEADLINE_STRETCH_SEC, FileGenerator.TYPE_OPT)
+ else:
+ self.deadline_type = None
+
self.rate_dist = self._get_dist(FileGenerator.RATE_SEC)
self.duration_dist = self._get_dist(FileGenerator.DURATION_SEC)
self.numnodes_dist = self._get_dist(FileGenerator.NODES_SEC)
self.software_dist = self._get_dist(FileGenerator.SOFTWARE_SEC)
- self.start_type = self.config.get(FileGenerator.START_DELAY_SEC, FileGenerator.TYPE_OPT)
- self.deadline_type = self.config.get(FileGenerator.DEADLINE_STRETCH_SEC, FileGenerator.TYPE_OPT)
def _get_start(self, type, lease = None):
@@ -199,31 +208,35 @@
max = float(max)
else:
min = max = None
-
+
if discrete:
- if dist_type == "uniform":
- if min != None:
- values = range(int(min), int(max) + 1)
- else:
- values = self.config.get(section, FileGenerator.VALUES_OPT).split()
+ if min != None:
+ values = range(int(min), int(max) + 1)
+ else:
+ values = self.config.get(section, FileGenerator.VALUES_OPT).split()
+
+ if dist_type == "uniform":
+ if discrete:
dist = stats.DiscreteUniformDistribution(values)
- else:
- if dist_type == "uniform":
+ else:
dist = stats.UniformDistribution(min, max)
- elif dist_type == "normal":
- mu = self.config.getfloat(section, FileGenerator.MEAN_OPT)
- sigma = self.config.getfloat(section, FileGenerator.STDEV_OPT)
- dist = stats.BoundedNormalDistribution(min,max,mu,sigma)
- elif dist_type == "bounded-pareto" or dist_type == "truncated-pareto":
- alpha = self.config.getfloat(section, FileGenerator.ALPHA_OPT)
- if self.config.has_option(section, FileGenerator.INVERT_OPT):
- invert = self.config.getboolean(section, FileGenerator.INVERT_OPT)
+ elif dist_type == "normal":
+ mu = self.config.getfloat(section, FileGenerator.MEAN_OPT)
+ sigma = self.config.getfloat(section, FileGenerator.STDEV_OPT)
+ dist = stats.BoundedNormalDistribution(min,max,mu,sigma)
+ elif dist_type == "bounded-pareto" or dist_type == "truncated-pareto":
+ alpha = self.config.getfloat(section, FileGenerator.ALPHA_OPT)
+ if self.config.has_option(section, FileGenerator.INVERT_OPT):
+ invert = self.config.getboolean(section, FileGenerator.INVERT_OPT)
+ else:
+ invert = False
+ if dist_type == "bounded-pareto":
+ dist = stats.BoundedParetoDistribution(min,max,alpha,invert)
+ else:
+ scale = self.config.getfloat(section, FileGenerator.SCALE_OPT)
+ if discrete:
+ dist = stats.DiscreteTruncatedParetoDistribution(values,scale,alpha,invert)
else:
- invert = False
- if dist_type == "bounded-pareto":
- dist = stats.BoundedParetoDistribution(min,max,alpha,invert)
- else:
- scale = self.config.getfloat(section, FileGenerator.SCALE_OPT)
dist = stats.TruncatedParetoDistribution(min,max,scale,alpha,invert)
@@ -358,38 +371,55 @@
class LWFAnnotationGenerator(FileGenerator):
- def __init__(self, lwffile, outfile, conffile):
+ def __init__(self, lwffile, nleases, outfile, conffile):
FileGenerator.__init__(self, outfile, conffile)
self.lwffile = lwffile
+ self.nleases = nleases
- def generate(self):
- lease_workload = LeaseWorkload.from_xml_file(self.lwffile)
- leases = lease_workload.get_leases()
- annotations = {}
+ def __gen_annotation(self, lease = None):
+ extra = {}
- for lease in leases:
- lease_id = lease.id
- extra = {}
+ start, delta = self._get_start(self.start_type, lease)
+ if start != None:
+ start = Timestamp(start)
+ extra["simul_start_delta"] = "%.2f" % delta
- start, delta = self._get_start(self.start_type, lease)
- if start != None:
- start = Timestamp(start)
- extra["simul_start_delta"] = "%.2f" % delta
-
deadline, tau = self._get_deadline(self.deadline_type, lease, start.requested)
if deadline != None:
extra["simul_deadline_tau"] = "%.2f" % tau
-
- software = self._get_software(lease)
-
- rate = self._get_rate(lease)
+ else:
+ deadline = None
+
+ software = self._get_software(lease)
+
+ rate = self._get_rate(lease)
- if rate != None:
- extra["simul_userrate"] = "%.2f" % rate
+ if rate != None:
+ extra["simul_userrate"] = "%.2f" % rate
+
+ if lease == None:
+ lease_id = None
+ else:
+ lease_id = lease.id
+
+ annotation = LeaseAnnotation(lease_id, start, deadline, software, extra)
+
+ return annotation
+
+ def generate(self):
+ if self.lwffile == None:
+ annotations = []
+ for i in xrange(1, self.nleases + 1):
+ annotation = self.__gen_annotation()
+ annotations.append(annotation)
+ else:
+ annotations = {}
+ lease_workload = LeaseWorkload.from_xml_file(self.lwffile)
+ leases = lease_workload.get_leases()
+ for lease in leases:
+ annotations = self.__gen_annotation(lease)
+ annotations[lease.id] = annotation
- annotation = LeaseAnnotation(lease_id, start, deadline, software, extra)
- annotations[lease_id] = annotation
-
attributes = self._get_attributes()
annotations = LeaseAnnotations(annotations, attributes)
More information about the Haizea-commit
mailing list