diff --git a/bin/logtool b/bin/logtool new file mode 100755 index 0000000000000000000000000000000000000000..4836d44411807e478e38a753dd98eb6a57989186 --- /dev/null +++ b/bin/logtool @@ -0,0 +1,102 @@ +#! /usr/bin/python + +def main(): + from pytools.log import LogManager + import sys + from optparse import OptionParser + + description = """Operate on data gathered during code runs. +FILE is a log saved from a code run. COMMANDS may be one of the +following: "list" to list the available variables, +"plot expr_x,expr_y" to plot a graph, +"datafile outfile expr_x,expr_y" to write out a data file.""" + parser = OptionParser(usage="%prog FILE COMMANDS FILE COMMANDS...", + description=description) + + parser.add_option("--units-x", + help="No description on the X axis", + action="store_true") + parser.add_option("--units-y", + help="No description on the Y axis", + action="store_true") + parser.add_option("--legend-expr", + help="Generate a legend from the expression", + action="store_true") + parser.add_option("--legend-descr", + help="Generate a legend from the description", + action="store_true") + parser.add_option("--title", + help="Set the title of a plot", + default="Log evaluation") + options, args = parser.parse_args() + + if len(args) < 1: + parser.print_help() + sys.exit(1) + + logmgr = LogManager() + + did_plot = False + + while args: + if args[0] == "list": + args.pop(0) + items = list(logmgr.quantity_buffers.iteritems()) + items.sort(lambda a,b: cmp(a[0], b[0])) + + col0_len = max(len(k) for k, v in items) + 1 + + for key, qbuf in items: + print "%s\t%s" % (key.ljust(col0_len), qbuf.quantity.description) + elif args[0] == "plot": + args.pop(0) + + expr_x, expr_y = args[0].split(",") + args.pop(0) + + from pylab import xlabel, ylabel, plot + (data_x, descr_x, unit_x), (data_y, descr_y, unit_y) = \ + logmgr.get_plot_data(expr_x, expr_y) + + if options.units_x: + xlabel(unit_x) + else: + xlabel("%s [%s]" % (descr_x, unit_x)) + if options.units_y: + ylabel(unit_y) + else: + ylabel("%s [%s]" % (descr_y, unit_y)) + + kwargs = {} + + if options.legend_expr: + kwargs["label"] = expr_y + if options.legend_descr: + kwargs["label"] = descr_y + + plot(data_x, data_y, hold=True, **kwargs) + + did_plot = True + elif args[0] == "datafile": + args.pop(0) + + expr_x, expr_y = args[0].split(",") + args.pop(0) + + logmgr.write_datafile(args[0], expr_x, expr_y) + args.pop(0) + else: + # not a known command, interpret as file name + logmgr.load(args[0]) + args.pop(0) + + if did_plot: + from pylab import show, title, legend + if options.legend_expr or options.legend_descr: + legend() + + title(options.title) + show() + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 770c333c33f7bb81d3a516f30dc5384030749546..d1b22790369b82a5cf1b78c44574f93f17696558 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ setup(name="pytools", version="0.10", description="A collection of tools for Python", author=u"Andreas Kloeckner", + scripts=["bin/logtool"], author_email="inform@tiker.net", license = "BSD, like Python", url="http://news.tiker.net/software/pytools", diff --git a/src/log.py b/src/log.py new file mode 100644 index 0000000000000000000000000000000000000000..9983fbab0765867a7bace0eb92d2459919463e04 --- /dev/null +++ b/src/log.py @@ -0,0 +1,352 @@ +from __future__ import division + + + + +# abstract logging interface -------------------------------------------------- +class LogQuantity: + def __init__(self, name, unit=None, description=None): + self.name = name + self.unit = unit + self.description = description + + def __call__(self): + raise NotImplementedError + +class PushLogQuantity(LogQuantity): + def __init__(self, name, unit=None, description=None): + LogQuantity.__init__(self, name, unit, description) + self.value = None + + def set(self, value): + if self.value is None: + self.value = value + else: + raise RuntimeError, "Push quantities may only be set once per cycle" + + def __call__(self): + return self.value + +class CallableLogQuantityAdapter(LogQuantity): + def __init__(self, callable, name, unit=None, description=None): + self.callable = callable + LogQuantity.__init__(self, name, unit, description) + + def __call__(self): + return self.callable() + + + + +# manager functionality ------------------------------------------------------- +class _QuantityBuffer: + def __init__(self, quantity, interval=1, buffer=[]): + self.quantity = quantity + self.interval = interval + self.buffer = buffer[:] + +def _join_by_first_of_tuple(list_of_iterables): + loi = [i.__iter__() for i in list_of_iterables] + if not loi: + return + key_vals = [iter.next() for iter in loi] + keys = [kv[0] for kv in key_vals] + values = [kv[1] for kv in key_vals] + target_key = max(keys) + + force_advance = False + + i = 0 + while True: + while keys[i] < target_key or force_advance: + try: + new_key, new_value = loi[i].next() + except StopIteration: + return + assert keys[i] < new_key + keys[i] = new_key + values[i] = new_value + if new_key > target_key: + target_key = new_key + + force_advance = False + + i += 1 + if i >= len(loi): + i = 0 + + if min(keys) == target_key: + yield target_key, values[:] + force_advance = True + + + + +class LogManager: + def __init__(self): + self.quantity_buffers = {} + self.tick_count = 0 + + def tick(self): + for qbuf in self.quantity_buffers.itervalues(): + if self.tick_count % qbuf.interval == 0: + qbuf.buffer.append((self.tick_count, qbuf.quantity())) + self.tick_count += 1 + + def add_quantity(self, quantity, interval=1): + """Add an object derived from L{LogQuantity} to this manager.""" + self.quantity_buffers[quantity.name] = _QuantityBuffer(quantity, interval) + + def get_expr_dataset(self, expression, description=None, unit=None): + """Return a triple C{(description, unit, buffer)} for the given expression. + + C{buffer} consists of a list of tuples C{(tick_nbr, value)}. + """ + try: + qbuf = self.quantity_buffers[expression] + except KeyError: + from pymbolic import parse, get_dependencies, evaluate, \ + var, substitute + parsed = parse(expression) + deps = [dep.name for dep in get_dependencies(parsed)] + + if unit is None: + unit = substitute(parsed, + dict((name, + var(self.quantity_buffers[name].quantity.unit)) + for name in deps)) + if description is None: + description = expression + return (description, + unit, + [(key, evaluate(parsed, + dict((name, value) + for name, value in zip(deps, values)) + )) + + for key, values in _join_by_first_of_tuple( + self.quantity_buffers[dep].buffer for dep in deps) + ]) + else: + return (description or qbuf.quantity.description, + unit or qbuf.quantity.unit, + qbuf.buffer) + + def get_joint_dataset(self, expressions): + """Return a joint data set for a list of expressions. + + @arg expressions: a list of either strings representing + expressions directly, or triples (descr, unit, expr). + In the former case, the description and the unit are + found automatically, if possible. In the latter case, + they are used as specified. + @return: A triple C{(descriptions, units, buffer)}, where + C{buffer} is a a list of C{[(tstep, (val_expr1, val_expr2,...)...]}. + """ + + # dubs is a list of (desc, unit, buffer) triples as + # returned by get_expr_dataset + dubs = [] + for expr in expressions: + if isinstance(expr, str): + dub = self.get_expr_dataset(expr) + else: + expr_descr, expr_unit, expr_str = expr + dub = get_expr_dataset( + expr_str, + description=expr_descr, + unit=expr_unit) + + dubs.append(dub) + + zipped_dubs = list(zip(*dubs)) + zipped_dubs[2] = list( + _join_by_first_of_tuple(zipped_dubs[2])) + + return zipped_dubs + + def save(self, filename): + save_buffers = dict( + (name, _QuantityBuffer( + LogQuantity( + qbuf.quantity.name, + qbuf.quantity.unit, + qbuf.quantity.description, + ), + qbuf.interval, + qbuf.buffer)) + for name, qbuf in self.quantity_buffers.iteritems()) + + from cPickle import dump, HIGHEST_PROTOCOL + dump(save_buffers, open(filename, "w"), protocol=HIGHEST_PROTOCOL) + + def load(self, filename): + from cPickle import load + self.quantity_buffers = load(open(filename)) + + def get_plot_data(self, expr_x, expr_y): + """Generate plot-ready data. + + @return: C{(data_x, descr_x, unit_x), (data_y, descr_y, unit_y)} + """ + (descr_x, descr_y), (unit_x, unit_y), data = \ + self.get_joint_dataset([expr_x, expr_y]) + _, stepless_data = zip(*data) + data_x, data_y = zip(*stepless_data) + return (data_x, descr_x, unit_x), \ + (data_y, descr_y, unit_y) + + def plot_gnuplot(self, gp, expr_x, expr_y, **kwargs): + """Plot data to Gnuplot.py. + + @arg gp: a Gnuplot.Gnuplot instance to which the plot is sent. + @arg expr_x: an allowed argument to L{get_joint_dataset}. + @arg expr_y: an allowed argument to L{get_joint_dataset}. + @arg kwargs: keyword arguments that are directly passed on to + C{Gnuplot.Data}. + """ + (data_x, descr_x, unit_x), (data_y, descr_y, unit_y) = \ + self.get_plot_data(expr_x, expr_y) + + gp.xlabel("%s [%s]" % (descr_x, unit_x)) + gp.ylabel("%s [%s]" % (descr_y, unit_y)) + gp.plot(Data(data_x, data_y, **kwargs)) + + def write_datafile(self, filename, expr_x, expr_y): + (data_x, label_x), (data_y, label_y) = self.get_plot_data( + expr_x, expr_y) + + outf = open(filename, "w") + outf.write("# %s [%s] vs. %s [%s]" % + (descr_x, unit_x, descr_y, unit_y)) + for dx, dy in zip(data_x, data_y): + outf.write("%s\t%s\n" % (repr(dx), repr(dy))) + outf.close() + + def plot_matplotlib(self, expr_x, expr_y): + from pylab import xlabel, ylabel, plot + + (data_x, descr_x, unit_x), (data_y, descr_y, unit_y) = \ + self.get_plot_data(expr_x, expr_y) + + xlabel("%s [%s]" % (descr_x, unit_x)) + ylabel("%s [%s]" % (descr_y, unit_y)) + xlabel(label_x) + ylabel(label_y) + plot(data_x, data_y) + + + +# actual data loggers --------------------------------------------------------- +class IntervalTimer(LogQuantity): + def __init__(self, name="interval", description=None): + LogQuantity.__init__(self, name, "s", description) + + self.elapsed = 0 + + def start(self): + from time import time + self.start_time = time() + + def stop(self): + from time import time + self.elapsed += time() - self.start_time + del self.start_time + + def __call__(self): + result = self.elapsed + self.elapsed = 0 + return result + + + + +class EventCounter(LogQuantity): + def __init__(self, name="interval", description=None): + LogQuantity.__init__(self, name, "1", description) + self.events = 0 + + def add(self, n=1): + self.events += n + + def transfer(self, counter): + self.events += counter.pop() + + def __call__(self): + result = self.events + self.events = 0 + return result + + + + +class TimestepCounter(LogQuantity): + def __init__(self, name="step"): + LogQuantity.__init__(self, name, "1", "Timesteps") + self.steps = 0 + + def __call__(self): + result = self.steps + self.steps += 1 + return result + + + + +class TimestepDuration(LogQuantity): + def __init__(self, name="t_step"): + LogQuantity.__init__(self, name, "s", "Time step duration") + + from time import time + self.last_start = time() + + def __call__(self): + from time import time + now = time() + result = now - self.last_start + self.last_start = now + return result + + + + +class WallTime(LogQuantity): + def __init__(self, name="t_wall"): + LogQuantity.__init__(self, name, "s", "Wall time") + + from time import time + self.start = time() + + def __call__(self): + from time import time + return time()-self.start + + + + +class SimulationTime(LogQuantity): + def __init__(self, dt, name="t_sim", start=0): + LogQuantity.__init__(self, name, "s", "Simulation Time") + self.dt = dt + self.t = 0 + + def set_dt(self, dt): + self.dt = dt + + def __call__(self): + result = self.t + self.t += self.dt + return result + + + + +def add_general_quantities(mgr, dt): + mgr.add_quantity(TimestepDuration()) + mgr.add_quantity(WallTime()) + mgr.add_quantity(SimulationTime(dt)) + mgr.add_quantity(TimestepCounter()) + + + +