[haizea-commit] r798 - in branches/1.1: bin src/haizea/cli src/haizea/common src/haizea/core src/haizea/core/frontends src/haizea/lwf
haizea-commit at mailman.cs.uchicago.edu
haizea-commit at mailman.cs.uchicago.edu
Tue Jun 1 13:33:35 CDT 2010
Author: borja
Date: 2010-06-01 13:33:29 -0500 (Tue, 01 Jun 2010)
New Revision: 798
Added:
branches/1.1/bin/haizea-lwf-stats
Modified:
branches/1.1/bin/haizea-lwf-generate
branches/1.1/src/haizea/cli/commands.py
branches/1.1/src/haizea/common/stats.py
branches/1.1/src/haizea/core/frontends/tracefile.py
branches/1.1/src/haizea/core/leases.py
branches/1.1/src/haizea/lwf/analysis.py
branches/1.1/src/haizea/lwf/generators.py
Log:
Utilization-based LWF generation (still needs some work, but mostly done)
Modified: branches/1.1/bin/haizea-lwf-generate
===================================================================
--- branches/1.1/bin/haizea-lwf-generate 2010-05-28 16:03:25 UTC (rev 797)
+++ branches/1.1/bin/haizea-lwf-generate 2010-06-01 18:33:29 UTC (rev 798)
@@ -3,5 +3,5 @@
from haizea.cli import commands
import sys
-c = commands.haizea_lwf_annotate(sys.argv)
+c = commands.haizea_lwf_generate(sys.argv)
c.run()
\ No newline at end of file
Added: branches/1.1/bin/haizea-lwf-stats
===================================================================
--- branches/1.1/bin/haizea-lwf-stats (rev 0)
+++ branches/1.1/bin/haizea-lwf-stats 2010-06-01 18:33:29 UTC (rev 798)
@@ -0,0 +1,7 @@
+#!/usr/bin/python
+
+from haizea.cli import commands
+import sys
+
+c = commands.haizea_lwf_stats(sys.argv)
+c.run()
\ No newline at end of file
Property changes on: branches/1.1/bin/haizea-lwf-stats
___________________________________________________________________
Added: svn:executable
+ *
Modified: branches/1.1/src/haizea/cli/commands.py
===================================================================
--- branches/1.1/src/haizea/cli/commands.py 2010-05-28 16:03:25 UTC (rev 797)
+++ branches/1.1/src/haizea/cli/commands.py 2010-06-01 18:33:29 UTC (rev 798)
@@ -25,6 +25,7 @@
from haizea.cli.optionparser import Option
from haizea.cli import Command
from haizea.lwf.generators import LWFGenerator, LWFAnnotationGenerator
+from haizea.lwf.analysis import LWFAnalyser
from mx.DateTime import TimeDelta, Parser
import haizea.common.defaults as defaults
import sys
@@ -367,7 +368,7 @@
self.optparser.add_option(Option("-c", "--conf", action="store", type="string", dest="conf",
help = """
...
- """))
+ """))
def run(self):
self.parse_options()
@@ -379,6 +380,34 @@
generator.generate()
+class haizea_lwf_stats(Command):
+
+ name = "haizea-lwf-stats"
+
+ def __init__(self, argv):
+ Command.__init__(self, argv)
+ self.optparser.add_option(Option("-i", "--in", action="store", type="string", dest="inf", required=True,
+ help = """
+ Input file
+ """))
+ self.optparser.add_option(Option("-l", "--utilization-length", action="store", type="string", dest="utilization_length",
+ help = """
+ Length of the utilization interval in format DD:HH:MM:SS. Default is until
+ the time the last lease is requested.
+ """))
+
+ def run(self):
+ self.parse_options()
+
+ infile = self.opt.inf
+ utilization_length = self.opt.utilization_length
+ if utilization_length != None:
+ utilization_length = Parser.DateTimeDeltaFromString(utilization_length)
+
+ analyser = LWFAnalyser(infile, utilization_length)
+
+ analyser.analyse()
+
class haizea_lwf_annotate(Command):
name = "haizea-lwf-annotate"
Modified: branches/1.1/src/haizea/common/stats.py
===================================================================
--- branches/1.1/src/haizea/common/stats.py 2010-05-28 16:03:25 UTC (rev 797)
+++ branches/1.1/src/haizea/common/stats.py 2010-06-01 18:33:29 UTC (rev 798)
@@ -137,6 +137,9 @@
DiscreteDistribution.__init__(self, values)
self.__distribution = UniformDistribution(0,1)
+ def seed(self, x):
+ self.__distribution.seed(x)
+
def get(self):
return self._get_from_prob(self.__distribution.get())
Modified: branches/1.1/src/haizea/core/frontends/tracefile.py
===================================================================
--- branches/1.1/src/haizea/core/frontends/tracefile.py 2010-05-28 16:03:25 UTC (rev 797)
+++ branches/1.1/src/haizea/core/frontends/tracefile.py 2010-06-01 18:33:29 UTC (rev 798)
@@ -40,11 +40,8 @@
# Requests is a list of lease requests
self.logger.info("Loading tracefile %s" % tracefile)
self.requests = None
- if tracefile.endswith(".swf"):
- self.requests = LeaseWorkload.from_swf_file(tracefile, self.starttime)
- elif tracefile.endswith(".lwf") or tracefile.endswith(".xml"):
- lease_workload = LeaseWorkload.from_xml_file(tracefile, self.starttime)
- self.requests = lease_workload.get_leases()
+ lease_workload = LeaseWorkload.from_xml_file(tracefile, self.starttime)
+ self.requests = lease_workload.get_leases()
if injectfile != None:
self.logger.info("Loading injection file %s" % injectfile)
Modified: branches/1.1/src/haizea/core/leases.py
===================================================================
--- branches/1.1/src/haizea/core/leases.py 2010-05-28 16:03:25 UTC (rev 797)
+++ branches/1.1/src/haizea/core/leases.py 2010-06-01 18:33:29 UTC (rev 798)
@@ -1573,7 +1573,11 @@
Argument:
lwf_file -- LWF file.
"""
- return cls.__from_xml_element(ET.parse(lwf_file).getroot().find("site"))
+ site_elem = ET.parse(lwf_file).getroot().find("site")
+ if site_elem == None:
+ return None # LWF file does not contain a <site> element
+ else:
+ return cls.__from_xml_element(site_elem)
@classmethod
def __from_xml_element(cls, element):
@@ -1620,7 +1624,7 @@
resource_str = resource_str.split()
numnodes = int(resource_str[0])
- resources = resource_str[1:]
+ resources = resource_str[1]
capacity = Capacity.from_resources_string(resources)
nodes = Nodes([(numnodes,capacity)])
@@ -1659,6 +1663,19 @@
return max_ninstances
+ def to_xml(self):
+ """Returns an ElementTree XML representation of the nodes
+
+ See the Haizea documentation for details on the
+ lease XML format.
+
+ """
+ site = ET.Element("site")
+ resource_types = ET.SubElement(site, "resource-types")
+ resource_types.set("names", " ".join(self.resource_types))
+ site.append(self.nodes.to_xml())
+
+ return site
class Nodes(object):
Modified: branches/1.1/src/haizea/lwf/analysis.py
===================================================================
--- branches/1.1/src/haizea/lwf/analysis.py 2010-05-28 16:03:25 UTC (rev 797)
+++ branches/1.1/src/haizea/lwf/analysis.py 2010-06-01 18:33:29 UTC (rev 798)
@@ -14,4 +14,45 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
-# -------------------------------------------------------------------------- #
\ No newline at end of file
+# -------------------------------------------------------------------------- #
+from mx.DateTime import DateTime
+from haizea.core.leases import LeaseWorkload, Site
+
+
+class LWFAnalyser(object):
+
+
+ def __init__(self, lwffile, utilization_length):
+ # Arbitrary start time
+ self.starttime = DateTime(2006,11,25,13)
+
+ self.workload = LeaseWorkload.from_xml_file(lwffile, self.starttime)
+ self.site = Site.from_lwf_file(lwffile)
+
+ if utilization_length == None:
+ self.utilization_length = self.workload.get_leases()[-1].submit_time - self.starttime
+ else:
+ self.utilization_length = utilization_length
+ print self.utilization_length
+
+ def analyse(self):
+ utilization = 0
+ for lease in self.workload.get_leases():
+ if lease.start.requested + lease.duration.requested > self.starttime + self.utilization_length:
+ duration = self.starttime + self.utilization_length - lease.start.requested
+ else:
+ duration = lease.duration.requested.seconds
+ for res in lease.requested_resources.values():
+ for i in range(1,res.get_ninstances("CPU") + 1):
+ utilization += res.get_quantity_instance("CPU", i) * duration
+
+ if self.site != None:
+ max_utilization = 0
+ duration = self.utilization_length.seconds
+ for res in self.site.nodes.get_all_nodes().values():
+ for i in range(1,res.get_ninstances("CPU") + 1):
+ max_utilization += res.get_quantity_instance("CPU", i) * duration
+
+
+ print "Utilization: %.2f%%" % (utilization / max_utilization)
+
\ No newline at end of file
Modified: branches/1.1/src/haizea/lwf/generators.py
===================================================================
--- branches/1.1/src/haizea/lwf/generators.py 2010-05-28 16:03:25 UTC (rev 797)
+++ branches/1.1/src/haizea/lwf/generators.py 2010-06-01 18:33:29 UTC (rev 798)
@@ -19,7 +19,7 @@
import haizea.common.stats as stats
from haizea.core.leases import LeaseWorkload, LeaseAnnotation, LeaseAnnotations, Timestamp, Lease,\
Capacity, Duration, UnmanagedSoftwareEnvironment,\
- DiskImageSoftwareEnvironment
+ DiskImageSoftwareEnvironment, Site
from haizea.common.utils import round_datetime_delta
from mx.DateTime import DateTimeDelta, TimeDelta, Parser
import ConfigParser
@@ -235,6 +235,8 @@
class LWFGenerator(FileGenerator):
+ SITE_OPT = "site"
+
NUMLEASES_SEC = "numleases"
NUMLEASES_TYPE_UTILIZATION = "utilization"
@@ -249,11 +251,8 @@
self.numleases_type = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.TYPE_OPT)
- if self.numleases_type == LWFGenerator.NUMLEASES_TYPE_INTERVAL:
- self.interval_dist = self._get_dist(LWFGenerator.NUMLEASES_SEC)
- else:
- self.interval_dist = None
-
+ self.interval_dist = self._get_dist(LWFGenerator.NUMLEASES_SEC)
+
def _get_interval(self):
if self.interval_dist == None:
return None
@@ -290,13 +289,17 @@
description = ET.SubElement(lwf, "description")
description.text = "Created with haizea-generate"
- #if self.opt.site != None:
- # site_elem = ET.parse(self.opt.site).getroot()
- # site_num_nodes = int(site_elem.find("nodes").find("node-set").get("numnodes"))
- # lwf.append(site_elem)
+ site = self.config.get(LWFGenerator.GENERAL_SEC, LWFGenerator.SITE_OPT)
+ if site.startswith("file:"):
+ sitefile = site.split(":")
+ site = Site.from_xml_file(sitefile[1])
+ else:
+ site = Site.from_resources_string(site)
+
+ lwf.append(site.to_xml())
time = TimeDelta(seconds=0)
- requests = ET.SubElement(lwf, "lease-requests")
+ requests = ET.SubElement(lwf, "lease-requests")
if self.numleases_type == LWFGenerator.NUMLEASES_TYPE_INTERVAL:
leases = []
@@ -308,7 +311,6 @@
for l in leases:
interval = TimeDelta(seconds=self._get_interval())
time += interval
- print interval, time
l.start.requested += time
lease_request = ET.SubElement(requests, "lease-request")
lease_request.set("arrival", str(time))
@@ -318,24 +320,41 @@
last_request = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.NUMLEASES_LAST_REQUEST_OPT)
last_request = Parser.DateTimeDeltaFromString(last_request)
- # TODO
- target_utilization = utilization
- accum_utilization = None
+ max_utilization = 0
+ for res in site.nodes.get_all_nodes().values():
+ for i in range(1,res.get_ninstances("CPU") + 1):
+ max_utilization += res.get_quantity_instance("CPU", i) * last_request.seconds
+ target_utilization = int(max_utilization * utilization)
+ accum_utilization = 0
+
leases = []
while accum_utilization < target_utilization:
- pass
- self.__gen_lease()
+ lease = self.__gen_lease()
+ leases.append(lease)
+ duration = lease.duration.requested.seconds
+ lease_utilization = 0
+ for res in lease.requested_resources.values():
+ for i in range(1,res.get_ninstances("CPU") + 1):
+ lease_utilization += res.get_quantity_instance("CPU", i) * duration
+ accum_utilization += lease_utilization
- # TODO: Set arrival times so they are evenly spaced
+ time = TimeDelta(seconds=0)
+ avg_interval = int(last_request.seconds / len(leases))
for l in leases:
+ interval = avg_interval + TimeDelta(seconds=self._get_interval())
+ time = max(time + interval, TimeDelta(seconds=0))
+ l.start.requested += time
lease_request = ET.SubElement(requests, "lease-request")
lease_request.set("arrival", str(time))
- lease_request.append(l.to_xml())
+ lease_request.append(l.to_xml())
-
- print ET.tostring(lwf)
+ tree = ET.ElementTree(lwf)
+
+ outfile = open(self.outfile, "w")
+ tree.write(outfile)
+ outfile.close()
class LWFAnnotationGenerator(FileGenerator):
More information about the Haizea-commit
mailing list