[haizea-commit] r468 - in trunk/src/haizea: cli common resourcemanager
haizea-commit at mailman.cs.uchicago.edu
haizea-commit at mailman.cs.uchicago.edu
Thu Aug 7 05:32:58 CDT 2008
Author: borja
Date: 2008-08-07 05:32:58 -0500 (Thu, 07 Aug 2008)
New Revision: 468
Modified:
trunk/src/haizea/cli/commands.py
trunk/src/haizea/common/config.py
trunk/src/haizea/common/utils.py
trunk/src/haizea/resourcemanager/accounting.py
trunk/src/haizea/resourcemanager/configfile.py
Log:
Adapt multiconfig to new configuration management. Save multiconfig attributes to accounting data object.
Modified: trunk/src/haizea/cli/commands.py
===================================================================
--- trunk/src/haizea/cli/commands.py 2008-08-07 10:18:58 UTC (rev 467)
+++ trunk/src/haizea/cli/commands.py 2008-08-07 10:32:58 UTC (rev 468)
@@ -17,8 +17,7 @@
# -------------------------------------------------------------------------- #
from haizea.resourcemanager.rm import ResourceManager
-from haizea.traces.generators import generateTrace, generateImages
-from haizea.common.utils import gen_traceinj_name, unpickle
+from haizea.common.utils import generate_config_name, unpickle
from haizea.resourcemanager.configfile import HaizeaConfig, HaizeaMultiConfig
from haizea.common.config import ConfigException
from haizea.cli.optionparser import OptionParser, Option
@@ -135,11 +134,11 @@
os.makedirs(etcdir)
for c in configs:
- profile = c.getProfile()
- tracefile = c.getTracefile()
- injfile = c.getInjectfile()
- name = gen_traceinj_name(tracefile, injfile)
- configfile = etcdir + "/%s_%s.conf" % (profile, name)
+ profile = c.get_attr("profile")
+ tracefile = c.get("tracefile")
+ injfile = c.get("injectionfile")
+ configname = generate_config_name(profile, tracefile, injfile)
+ configfile = etcdir + "/%s.conf" % configname
fc = open(configfile, "w")
c.config.write(fc)
fc.close()
@@ -163,7 +162,7 @@
print "You can download them at http://www.makotemplates.org/"
exit(1)
- configs = multiconfig.get_configs_to_run()
+ configs = multiconfig.get_configs()
etcdir = os.path.abspath(opt.confdir)
if not os.path.exists(etcdir):
@@ -171,14 +170,14 @@
templatedata = []
for c in configs:
- profile = c.getProfile()
- tracefile = c.getTracefile()
- injfile = c.getInjectfile()
- datafile = c.getDataFile()
- name = gen_traceinj_name(tracefile, injfile)
+ profile = c.get_attr("profile")
+ tracefile = c.get("tracefile")
+ injfile = c.get("injectionfile")
+ datafile = c.get("datafile")
+ configname = generate_config_name(profile, tracefile, injfile)
if not opt.onlymissing or not os.path.exists(datafile):
- configfile = etcdir + "/%s_%s.conf" % (profile, name)
- templatedata.append((profile, name, configfile))
+ configfile = etcdir + "/%s.conf" % configname
+ templatedata.append((configname, configfile))
template = Template(filename=opt.template)
print template.render(configs=templatedata, etcdir=etcdir)
Modified: trunk/src/haizea/common/config.py
===================================================================
--- trunk/src/haizea/common/config.py 2008-08-07 10:18:58 UTC (rev 467)
+++ trunk/src/haizea/common/config.py 2008-08-07 10:32:58 UTC (rev 468)
@@ -62,7 +62,7 @@
def __init__(self, config, sections):
self.config = config
self.sections = sections
- self.__options = {}
+ self._options = {}
self.__load_all()
@@ -136,10 +136,10 @@
if not value in opt.valid:
raise ConfigException, "Invalid value specified for '%s.%s'. Valid values are %s" % (secname, optname, opt.valid)
- self.__options[opt.getter] = value
+ self._options[opt.getter] = value
def get(self, opt):
- return self.__options[opt]
+ return self._options[opt]
@classmethod
def from_file(cls, configfile):
Modified: trunk/src/haizea/common/utils.py
===================================================================
--- trunk/src/haizea/common/utils.py 2008-08-07 10:18:58 UTC (rev 467)
+++ trunk/src/haizea/common/utils.py 2008-08-07 10:32:58 UTC (rev 468)
@@ -21,20 +21,17 @@
from cPickle import dump, load, HIGHEST_PROTOCOL
from datetime import datetime
-def gen_traceinj_name(tracefile, injectedfile):
+def generate_config_name(profile, tracefile, injectedfile):
tracename=tracefile.split("/")[-1].split(".")[0]
- if injectedfile != None:
+ if injectedfile != None and injectedfile != "None":
injectname=injectedfile.split("/")[-1].split(".")[0]
name = tracename + "+" + injectname
else:
name = tracename
-
+
+ name = profile + "_" + name
return name
-
-def genDataDirName(profile, tracefile, injectedfile):
- name = gen_traceinj_name(tracefile, injectedfile)
- return profile + "/" + name + "/"
def roundDateTimeDelta(d):
return DateTime.DateTimeDelta(d.day, d.hour, d.minute, int(ceil(d.second)))
Modified: trunk/src/haizea/resourcemanager/accounting.py
===================================================================
--- trunk/src/haizea/resourcemanager/accounting.py 2008-08-07 10:18:58 UTC (rev 467)
+++ trunk/src/haizea/resourcemanager/accounting.py 2008-08-07 10:32:58 UTC (rev 468)
@@ -36,6 +36,9 @@
# Lease data
self.leases = {}
+ # Attributes
+ self.attrs = {}
+
def get_waiting_times(self):
waiting_times = {}
for lease_id in self.leases:
@@ -58,6 +61,10 @@
self.rm = rm
self.datafile = datafile
self.starttime = None
+
+ attrs = self.rm.config.get_attrs()
+ for attr in attrs:
+ self.data.attrs[attr] = self.rm.config.get_attr(attr)
def create_counter(self, counter_id, avgtype, initial=0):
self.data.counters[counter_id] = initial
Modified: trunk/src/haizea/resourcemanager/configfile.py
===================================================================
--- trunk/src/haizea/resourcemanager/configfile.py 2008-08-07 10:18:58 UTC (rev 467)
+++ trunk/src/haizea/resourcemanager/configfile.py 2008-08-07 10:32:58 UTC (rev 468)
@@ -16,11 +16,13 @@
# limitations under the License. #
# -------------------------------------------------------------------------- #
-from haizea.common.config import Section, Option, Config, OPTTYPE_INT, OPTTYPE_FLOAT, OPTTYPE_STRING, OPTTYPE_BOOLEAN, OPTTYPE_DATETIME, OPTTYPE_TIMEDELTA
+from haizea.common.config import ConfigException, Section, Option, Config, OPTTYPE_INT, OPTTYPE_FLOAT, OPTTYPE_STRING, OPTTYPE_BOOLEAN, OPTTYPE_DATETIME, OPTTYPE_TIMEDELTA
+from haizea.common.utils import generate_config_name
import haizea.common.constants as constants
-import haizea.common.stats as stats
import os.path
+import sys
from mx.DateTime import TimeDelta
+import ConfigParser
class HaizeaConfig(Config):
def __init__(self, config):
@@ -108,8 +110,8 @@
utilization data, etc.). If omitted, no data will be saved.
"""),
- Option(name = "profile",
- getter = "profile",
+ Option(name = "attributes",
+ getter = "attributes",
type = OPTTYPE_STRING,
required = False,
doc = """
@@ -231,6 +233,19 @@
- mem: only the memory must be transferred
- mem+disk: both the memory and the VM disk image must be
transferred
+ """),
+
+ Option(name = "non-schedulable-interval",
+ getter = "non-schedulable-interval",
+ type = OPTTYPE_TIMEDELTA,
+ required = False,
+ default = TimeDelta(seconds=10),
+ doc = """
+ The minimum amount of time that must pass between
+ when a request is scheduled to when it can actually start.
+ The default should be good for most configurations, but
+ may need to be increased if you're dealing with exceptionally
+ high loads.
""")
]
@@ -530,80 +545,62 @@
Rate at which VMs are estimated to suspend (in MB of
memory per second)
"""),
-
- Option(name = "non-schedulable-interval",
- getter = "non-schedulable-interval",
- type = OPTTYPE_TIMEDELTA,
- required = False,
- default = TimeDelta(seconds=10),
- doc = """
- The minimum amount of time that must pass between
- when a request is scheduled to when it can actually start.
- The default should be good for most configurations, but
- may need to be increased if you're dealing with exceptionally
- high loads.
- """)
]
sections.append(opennebula)
Config.__init__(self, config, sections)
+ self.attrs = {}
+ if self._options["attributes"] != None:
+ self.attrs = {}
+ attrs = self._options["attributes"].split(";")
+ for attr in attrs:
+ (k,v) = attr.split("=")
+ self.attrs[k] = v
+
+ def get_attr(self, attr):
+ return self.attrs[attr]
+
+ def get_attrs(self):
+ return self.attrs.keys()
class HaizeaMultiConfig(Config):
+
+ MULTI_SEC = "multi"
+ COMMON_SEC = "common"
+ TRACEDIR_OPT = "tracedir"
+ TRACEFILES_OPT = "tracefiles"
+ INJDIR_OPT = "injectiondir"
+ INJFILES_OPT = "injectionfiles"
+ DATADIR_OPT = "datadir"
+
def __init__(self, config):
- Config.__init__(self, config)
+ # TODO: Define "multi" section as a Section object
+ Config.__init__(self, config, [])
- def getProfiles(self):
+ def get_profiles(self):
sections = set([s.split(":")[0] for s in self.config.sections()])
# Remove multi and common sections
- sections.difference_update([constants.COMMON_SEC, constants.MULTI_SEC])
+ sections.difference_update([self.COMMON_SEC, self.MULTI_SEC])
return list(sections)
-
- def getProfilesSubset(self, sec):
- profiles = self.config.get(sec, constants.PROFILES_OPT)
- if profiles == "ALL":
- profiles = self.getProfiles()
- else:
- profiles = profiles.split()
- return profiles
- def getTracesSubset(self, sec):
- traces = self.config.get(sec, constants.TRACES_OPT)
- if traces == "ALL":
- traces = [os.path.basename(t) for t in self.getTracefiles()]
- else:
- traces = traces.split()
-
- return traces
-
- def getInjSubset(self, sec):
- injs = self.config.get(sec, constants.INJS_OPT)
- if injs == "ALL":
- injs = [os.path.basename(t) for t in self.getInjectfiles() if t!=None]
- injs.append(None)
- elif injs == "NONE":
- injs = [None]
- else:
- injs = injs.split()
- return injs
-
- def getTracefiles(self):
- dir = self.config.get(constants.MULTI_SEC, constants.TRACEDIR_OPT)
- traces = self.config.get(constants.MULTI_SEC, constants.TRACEFILES_OPT).split()
+ def get_trace_files(self):
+ dir = self.config.get(self.MULTI_SEC, self.TRACEDIR_OPT)
+ traces = self.config.get(self.MULTI_SEC, self.TRACEFILES_OPT).split()
return [dir + "/" + t for t in traces]
- def getInjectfiles(self):
- dir = self.config.get(constants.MULTI_SEC, constants.INJDIR_OPT)
- inj = self.config.get(constants.MULTI_SEC, constants.INJFILES_OPT).split()
+ def get_inject_files(self):
+ dir = self.config.get(self.MULTI_SEC, self.INJDIR_OPT)
+ inj = self.config.get(self.MULTI_SEC, self.INJFILES_OPT).split()
inj = [dir + "/" + i for i in inj]
inj.append(None)
return inj
- def getConfigs(self):
- profiles = self.getProfiles()
- tracefiles = self.getTracefiles()
- injectfiles = self.getInjectfiles()
+ def get_configs(self):
+ profiles = self.get_profiles()
+ tracefiles = self.get_trace_files()
+ injectfiles = self.get_inject_files()
configs = []
for profile in profiles:
@@ -622,54 +619,37 @@
profileconfig.set(s_noprefix, item[0], item[1])
# The tracefile section may have not been created
- if not profileconfig.has_section(constants.TRACEFILE_SEC):
- profileconfig.add_section(constants.TRACEFILE_SEC)
+ if not profileconfig.has_section("tracefile"):
+ profileconfig.add_section("tracefile")
# Add tracefile option
- profileconfig.set(constants.TRACEFILE_SEC, constants.TRACEFILE_OPT, tracefile)
+ profileconfig.set("tracefile", "tracefile", tracefile)
# Add injected file option
if injectfile == None:
inj = "None"
else:
inj = injectfile
- profileconfig.set(constants.TRACEFILE_SEC, constants.INJFILE_OPT, inj)
+ profileconfig.set("tracefile", "injectionfile", inj)
- # Add datadir option
- datadirname = genDataDirName(profile, tracefile, injectfile)
- basedatadir = self.config.get(constants.MULTI_SEC, constants.BASEDATADIR_OPT)
- # TODO: Change this so there will be a single directory with all the
- # data files, instead of multiple directories
- datafile = basedatadir + "/" + datadirname + "/haizea.dat"
- profileconfig.set(constants.GENERAL_SEC, constants.DATAFILE_OPT, datadir)
+ # Add datafile option
+ datadir = self.config.get(self.MULTI_SEC, self.DATADIR_OPT)
+ datafilename = generate_config_name(profile, tracefile, injectfile)
+ datafile = datadir + "/" + datafilename + ".dat"
+ profileconfig.set("general", "datafile", datafile)
- # Set profile option (only used internally)
- profileconfig.set(constants.GENERAL_SEC, constants.PROFILE_OPT, profile)
+ # Set "attributes" option (only used internally)
+ attrs = {"profile":profile}
+ # TODO: Load additional attributes from trace/injfiles
+ attrs_str = ",".join(["%s=%s" % (k,v) for (k,v) in attrs.items()])
+ profileconfig.set("general", "attributes", attrs_str)
- c = RMConfig(profileconfig)
+ try:
+ c = HaizeaConfig(profileconfig)
+ except ConfigException, msg:
+ print >> sys.stderr, "Error in configuration file:"
+ print >> sys.stderr, msg
+ exit(1)
configs.append(c)
- return configs
-
-
- def getConfigsToRun(self):
- configs = self.getConfigs()
-
- # TODO: Come up with a new way to filter what gets run or not
- #profiles = self.getProfilesSubset(constants.RUN_SEC)
- #traces = self.getTracesSubset(constants.RUN_SEC)
- #injs = self.getInjSubset(constants.RUN_SEC)
-
-# confs = []
-# for c in configs:
-# p = c.getProfile()
-# t = os.path.basename(c.getTracefile())
-# i = c.getInjectfile()
-# if i != None:
-# i = os.path.basename(i)
-#
-# if p in profiles and t in traces and i in injs:
-# confs.append(c)
-#
-# return confs
return configs
\ No newline at end of file
More information about the Haizea-commit
mailing list