1# Copyright (c) 2013 ARM Limited
2# All rights reserved
3#
4# The license below extends only to copyright in the software and shall
5# not be construed as granting a license to any other intellectual
6# property including but not limited to intellectual property relating
7# to a hardware implementation of the functionality of the software
8# licensed hereunder.  You may use the software subject to the license
9# terms below provided that you ensure that this notice is replicated
10# unmodified and in its entirety in all distributions of the software,
11# modified or unmodified, in source code or in binary form.
12#
13# Redistribution and use in source and binary forms, with or without
14# modification, are permitted provided that the following conditions are
15# met: redistributions of source code must retain the above copyright
16# notice, this list of conditions and the following disclaimer;
17# redistributions in binary form must reproduce the above copyright
18# notice, this list of conditions and the following disclaimer in the
19# documentation and/or other materials provided with the distribution;
20# neither the name of the copyright holders nor the names of its
21# contributors may be used to endorse or promote products derived from
22# this software without specific prior written permission.
23#
24# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35#
36# Authors: Andrew Bardsley
37
38import parse
39import colours
40from colours import unknownColour
41from point import Point
42import re
43import blobs
44from time import time as wall_time
45import os
46
47id_parts = "TSPLFE"
48
49all_ids = set(id_parts)
50no_ids = set([])
51
52class BlobDataSelect(object):
53    """Represents which data is displayed for Ided object"""
54    def __init__(self):
55        # Copy all_ids
56        self.ids = set(all_ids)
57
58    def __and__(self, rhs):
59        """And for filtering"""
60        ret = BlobDataSelect()
61        ret.ids = self.ids.intersection(rhs.ids)
62        return ret
63
64class BlobVisualData(object):
65    """Super class for block data colouring"""
66    def to_striped_block(self, select):
67        """Return an array of colours to use for a striped block"""
68        return unknownColour
69
70    def get_inst(self):
71        """Get an instruction Id (if any) from this data"""
72        return None
73
74    def get_line(self):
75        """Get a line Id (if any) from this data"""
76        return None
77
78    def __repr__(self):
79        return self.__class__.__name__ + '().from_string(' + \
80            self.__str__() + ')'
81
82    def __str__(self):
83        return ''
84
85class Id(BlobVisualData):
86    """A line or instruction id"""
87    def __init__(self):
88        self.isFault = False
89        self.threadId = 0
90        self.streamSeqNum = 0
91        self.predictionSeqNum = 0
92        self.lineSeqNum = 0
93        self.fetchSeqNum = 0
94        self.execSeqNum = 0
95
96    def as_list(self):
97        return [self.threadId, self.streamSeqNum, self.predictionSeqNum,
98            self.lineSeqNum, self.fetchSeqNum, self.execSeqNum]
99
100    def __cmp__(self, right):
101        return cmp(self.as_list(), right.as_list())
102
103    def from_string(self, string):
104        m = re.match('^(F;)?(\d+)/(\d+)\.(\d+)/(\d+)(/(\d+)(\.(\d+))?)?',
105            string)
106
107        def seqnum_from_string(string):
108            if string is None:
109                return 0
110            else:
111                return int(string)
112
113        if m is None:
114            print 'Invalid Id string', string
115        else:
116            elems = m.groups()
117
118            if elems[0] is not None:
119                self.isFault = True
120            else:
121                self.isFault = False
122
123            self.threadId = seqnum_from_string(elems[1])
124            self.streamSeqNum = seqnum_from_string(elems[2])
125            self.predictionSeqNum = seqnum_from_string(elems[3])
126            self.lineSeqNum = seqnum_from_string(elems[4])
127            self.fetchSeqNum = seqnum_from_string(elems[6])
128            self.execSeqNum = seqnum_from_string(elems[8])
129        return self
130
131    def get_inst(self):
132        if self.fetchSeqNum != 0:
133            return self
134        else:
135            return None
136
137    def get_line(self):
138        return self
139
140    def __str__(self):
141        """Returns the usual id T/S.P/L/F.E string"""
142        return (
143            str(self.threadId) + '/' +
144            str(self.streamSeqNum) + '.' +
145            str(self.predictionSeqNum) + '/' +
146            str(self.lineSeqNum) + '/' +
147            str(self.fetchSeqNum) + '.' +
148            str(self.execSeqNum))
149
150    def to_striped_block(self, select):
151        ret = []
152
153        if self.isFault:
154            ret.append(colours.faultColour)
155
156        if 'T' in select.ids:
157            ret.append(colours.number_to_colour(self.threadId))
158        if 'S' in select.ids:
159            ret.append(colours.number_to_colour(self.streamSeqNum))
160        if 'P' in select.ids:
161            ret.append(colours.number_to_colour(self.predictionSeqNum))
162        if 'L' in select.ids:
163            ret.append(colours.number_to_colour(self.lineSeqNum))
164        if self.fetchSeqNum != 0 and 'F' in select.ids:
165            ret.append(colours.number_to_colour(self.fetchSeqNum))
166        if self.execSeqNum != 0 and 'E' in select.ids:
167            ret.append(colours.number_to_colour(self.execSeqNum))
168
169        if len(ret) == 0:
170            ret = [colours.unknownColour]
171
172        if self.isFault:
173            ret.append(colours.faultColour)
174
175        return ret
176
177class Branch(BlobVisualData):
178    """Branch data new stream and prediction sequence numbers, a branch
179    reason and a new PC"""
180    def __init__(self):
181        self.newStreamSeqNum = 0
182        self.newPredictionSeqNum = 0
183        self.newPC = 0
184        self.reason = "NoBranch"
185        self.id = Id()
186
187    def from_string(self, string):
188        m = re.match('^(\w+);(\d+)\.(\d+);([0-9a-fA-Fx]+);(.*)$', string)
189
190        if m is not None:
191            self.reason, newStreamSeqNum, newPredictionSeqNum, \
192                newPC, id = m.groups()
193
194            self.newStreamSeqNum = int(newStreamSeqNum)
195            self.newPredictionSeqNum = int(newPredictionSeqNum)
196            self.newPC = int(newPC, 0)
197            self.id = special_view_decoder(Id)(id)
198            # self.branch = special_view_decoder(Branch)(branch)
199        else:
200            print "Bad Branch data:", string
201        return self
202
203    def to_striped_block(self, select):
204        return [colours.number_to_colour(self.newStreamSeqNum),
205            colours.number_to_colour(self.newPredictionSeqNum),
206            colours.number_to_colour(self.newPC)]
207
208class Counts(BlobVisualData):
209    """Treat the input data as just a /-separated list of count values (or
210    just a single value)"""
211    def __init__(self):
212        self.counts = []
213
214    def from_string(self, string):
215        self.counts = map(int, re.split('/', string))
216        return self
217
218    def to_striped_block(self, select):
219        return map(colours.number_to_colour, self.counts)
220
221class Colour(BlobVisualData):
222    """A fixed colour block, used for special colour decoding"""
223    def __init__(self, colour):
224        self.colour = colour
225
226    def to_striped_block(self, select):
227        return [self.colour]
228
229class DcacheAccess(BlobVisualData):
230    """Data cache accesses [RW];id"""
231    def __init__(self):
232        self.direc = 'R'
233        self.id = Id()
234
235    def from_string(self, string):
236        self.direc, id = re.match('^([RW]);([^;]*);.*$', string).groups()
237        self.id.from_string(id)
238        return self
239
240    def get_inst(self):
241        return self.id
242
243    def to_striped_block(self, select):
244        if self.direc == 'R':
245            direc_colour = colours.readColour
246        elif self.direc == 'R':
247            direc_colour = colours.writeColour
248        else:
249            direc_colour = colours.errorColour
250        return [direc_colour] + self.id.to_striped_block(select)
251
252class ColourPattern(object):
253    """Super class for decoders that make 2D grids rather than just single
254    striped blocks"""
255    def elems(self):
256        return []
257
258    def to_striped_block(self, select):
259        return [[[colours.errorColour]]]
260
261def special_view_decoder(class_):
262    """Generate a decode function that checks for special character
263    arguments first (and generates a fixed colour) before building a
264    BlobVisualData of the given class"""
265    def decode(symbol):
266        if symbol in special_state_colours:
267            return Colour(special_state_colours[symbol])
268        else:
269            return class_().from_string(symbol)
270    return decode
271
272class TwoDColours(ColourPattern):
273    """A 2D grid pattern decoder"""
274    def __init__(self, blockss):
275        self.blockss = blockss
276
277    @classmethod
278    def decoder(class_, elemClass, dataName):
279        """Factory for making decoders for particular block types"""
280        def decode(pairs):
281            if dataName not in pairs:
282                print 'TwoDColours: no event data called:', \
283                    dataName, 'in:', pairs
284                return class_([[Colour(colours.errorColour)]])
285            else:
286                parsed = parse.list_parser(pairs[dataName])
287                return class_(parse.map2(special_view_decoder(elemClass), \
288                    parsed))
289        return decode
290
291    @classmethod
292    def indexed_decoder(class_, elemClass, dataName, picPairs):
293        """Factory for making decoders for particular block types but
294        where the list elements are pairs of (index, data) and
295        strip and stripelems counts are picked up from the pair
296        data on the decoder's picture file.  This gives a 2D layout
297        of the values with index 0 at strip=0, elem=0 and index 1
298        at strip=0, elem=1"""
299        def decode(pairs):
300            if dataName not in pairs:
301                print 'TwoDColours: no event data called:', \
302                    dataName, 'in:', pairs
303                return class_([[Colour(colours.errorColour)]])
304            else:
305                strips = int(picPairs['strips'])
306                strip_elems = int(picPairs['stripelems'])
307
308                raw_iv_pairs = pairs[dataName]
309
310                parsed = parse.parse_indexed_list(raw_iv_pairs)
311
312                array = [[Colour(colours.emptySlotColour)
313                    for i in xrange(0, strip_elems)]
314                    for j in xrange(0, strips)]
315
316                for index, value in parsed:
317                    try:
318                        array[index % strips][index / strips] = \
319                            special_view_decoder(elemClass)(value)
320                    except:
321                        print "Element out of range strips: %d," \
322                            " stripelems %d, index: %d" % (strips,
323                            strip_elems, index)
324
325                # return class_(array)
326                return class_(array)
327        return decode
328
329    def elems(self):
330        """Get a flat list of all elements"""
331        ret = []
332        for blocks in self.blockss:
333            ret += blocks
334        return ret
335
336    def to_striped_block(self, select):
337        return parse.map2(lambda d: d.to_striped_block(select), self.blockss)
338
339class FrameColours(ColourPattern):
340    """Decode to a 2D grid which has a single occupied row from the event
341    data and some blank rows forming a frame with the occupied row as a
342    'title' coloured stripe"""
343    def __init__(self, block, numBlankSlots):
344        self.numBlankSlots = numBlankSlots
345        self.block = block
346
347    @classmethod
348    def decoder(class_, elemClass, numBlankSlots, dataName):
349        """Factory for element type"""
350        def decode(pairs):
351            if dataName not in pairs:
352                print 'FrameColours: no event data called:', dataName, \
353                    'in:', pairs
354                return class_([Colour(colours.errorColour)])
355            else:
356                parsed = parse.list_parser(pairs[dataName])
357                return class_(special_view_decoder(elemClass)
358                    (parsed[0][0]), numBlankSlots)
359        return decode
360
361    def elems(self):
362        return [self.block]
363
364    def to_striped_block(self, select):
365        return ([[self.block.to_striped_block(select)]] +
366            (self.numBlankSlots * [[[colours.backgroundColour]]]))
367
368special_state_colours = {
369    'U': colours.unknownColour,
370    'B': colours.blockedColour,
371    '-': colours.bubbleColour,
372    '': colours.emptySlotColour,
373    'E': colours.emptySlotColour,
374    'R': colours.reservedSlotColour,
375    'X': colours.errorColour,
376    'F': colours.faultColour,
377    'r': colours.readColour,
378    'w': colours.writeColour
379    }
380
381special_state_names = {
382    'U': '(U)nknown',
383    'B': '(B)locked',
384    '-': '(-)Bubble',
385    '': '()Empty',
386    'E': '(E)mpty',
387    'R': '(R)eserved',
388    'X': '(X)Error',
389    'F': '(F)ault',
390    'r': '(r)ead',
391    'w': '(w)rite'
392    }
393
394special_state_chars = special_state_colours.keys()
395
396# The complete set of available block data types
397decoder_element_classes = {
398    'insts': Id,
399    'lines': Id,
400    'branch': Branch,
401    'dcache': DcacheAccess,
402    'counts': Counts
403    }
404
405indexed_decoder_element_classes = {
406    'indexedCounts' : Counts
407    }
408
409def find_colour_decoder(stripSpace, decoderName, dataName, picPairs):
410    """Make a colour decoder from some picture file blob attributes"""
411    if decoderName == 'frame':
412        return FrameColours.decoder(Counts, stripSpace, dataName)
413    elif decoderName in decoder_element_classes:
414        return TwoDColours.decoder(decoder_element_classes[decoderName],
415            dataName)
416    elif decoderName in indexed_decoder_element_classes:
417        return TwoDColours.indexed_decoder(
418            indexed_decoder_element_classes[decoderName], dataName, picPairs)
419    else:
420        return None
421
422class IdedObj(object):
423    """An object identified by an Id carrying paired data.
424    The super class for Inst and Line"""
425
426    def __init__(self, id, pairs={}):
427        self.id = id
428        self.pairs = pairs
429
430    def __cmp__(self, right):
431        return cmp(self.id, right.id)
432
433    def table_line(self):
434        """Represent the object as a list of table row data"""
435        return []
436
437    # FIXME, add a table column titles?
438
439    def __repr__(self):
440        return ' '.join(self.table_line())
441
442class Inst(IdedObj):
443    """A non-fault instruction"""
444    def __init__(self, id, disassembly, addr, pairs={}):
445        super(Inst,self).__init__(id, pairs)
446        if 'nextAddr' in pairs:
447            self.nextAddr = int(pairs['nextAddr'], 0)
448            del pairs['nextAddr']
449        else:
450            self.nextAddr = None
451        self.disassembly = disassembly
452        self.addr = addr
453
454    def table_line(self):
455        if self.nextAddr is not None:
456            addrStr = '0x%x->0x%x' % (self.addr, self.nextAddr)
457        else:
458            addrStr = '0x%x' % self.addr
459        ret = [addrStr, self.disassembly]
460        for name, value in self.pairs.iteritems():
461            ret.append("%s=%s" % (name, str(value)))
462        return ret
463
464class InstFault(IdedObj):
465    """A fault instruction"""
466    def __init__(self, id, fault, addr, pairs={}):
467        super(InstFault,self).__init__(id, pairs)
468        self.fault = fault
469        self.addr = addr
470
471    def table_line(self):
472        ret = ["0x%x" % self.addr, self.fault]
473        for name, value in self.pairs:
474            ret.append("%s=%s", name, str(value))
475        return ret
476
477class Line(IdedObj):
478    """A fetched line"""
479    def __init__(self, id, vaddr, paddr, size, pairs={}):
480        super(Line,self).__init__(id, pairs)
481        self.vaddr = vaddr
482        self.paddr = paddr
483        self.size = size
484
485    def table_line(self):
486        ret = ["0x%x/0x%x" % (self.vaddr, self.paddr), "%d" % self.size]
487        for name, value in self.pairs:
488            ret.append("%s=%s", name, str(value))
489        return ret
490
491class LineFault(IdedObj):
492    """A faulting line"""
493    def __init__(self, id, fault, vaddr, pairs={}):
494        super(LineFault,self).__init__(id, pairs)
495        self.vaddr = vaddr
496        self.fault = fault
497
498    def table_line(self):
499        ret = ["0x%x" % self.vaddr, self.fault]
500        for name, value in self.pairs:
501            ret.append("%s=%s", name, str(value))
502        return ret
503
504class BlobEvent(object):
505    """Time event for a single blob"""
506    def __init__(self, unit, time, pairs = {}):
507        # blob's unit name
508        self.unit = unit
509        self.time = time
510        # dict of picChar (blob name) to visual data
511        self.visuals = {}
512        # Miscellaneous unparsed MinorTrace line data
513        self.pairs = pairs
514        # Non-MinorTrace debug printout for this unit at this time
515        self.comments = []
516
517    def find_ided_objects(self, model, picChar, includeInstLines):
518        """Find instructions/lines mentioned in the blob's event
519        data"""
520        ret = []
521        if picChar in self.visuals:
522            blocks = self.visuals[picChar].elems()
523            def find_inst(data):
524                instId = data.get_inst()
525                lineId = data.get_line()
526                if instId is not None:
527                    inst = model.find_inst(instId)
528                    line = model.find_line(instId)
529                    if inst is not None:
530                        ret.append(inst)
531                    if includeInstLines and line is not None:
532                        ret.append(line)
533                elif lineId is not None:
534                    line = model.find_line(lineId)
535                    if line is not None:
536                        ret.append(line)
537            map(find_inst, blocks)
538        return sorted(ret)
539
540class BlobModel(object):
541    """Model bringing together blob definitions and parsed events"""
542    def __init__(self, unitNamePrefix=''):
543        self.blobs = []
544        self.unitNameToBlobs = {}
545        self.unitEvents = {}
546        self.clear_events()
547        self.picSize = Point(20,10)
548        self.lastTime = 0
549        self.unitNamePrefix = unitNamePrefix
550
551    def clear_events(self):
552        """Drop all events and times"""
553        self.lastTime = 0
554        self.times = []
555        self.insts = {}
556        self.lines = {}
557        self.numEvents = 0
558
559        for unit, events in self.unitEvents.iteritems():
560            self.unitEvents[unit] = []
561
562    def add_blob(self, blob):
563        """Add a parsed blob to the model"""
564        self.blobs.append(blob)
565        if blob.unit not in self.unitNameToBlobs:
566            self.unitNameToBlobs[blob.unit] = []
567
568        self.unitNameToBlobs[blob.unit].append(blob)
569
570    def add_inst(self, inst):
571        """Add a MinorInst instruction definition to the model"""
572        # Is this a non micro-op instruction.  Microops (usually) get their
573        #   fetchSeqNum == 0 varient stored first
574        macroop_key = (inst.id.fetchSeqNum, 0)
575        full_key = (inst.id.fetchSeqNum, inst.id.execSeqNum)
576
577        if inst.id.execSeqNum != 0 and macroop_key not in self.insts:
578            self.insts[macroop_key] = inst
579
580        self.insts[full_key] = inst
581
582    def find_inst(self, id):
583        """Find an instruction either as a microop or macroop"""
584        macroop_key = (id.fetchSeqNum, 0)
585        full_key = (id.fetchSeqNum, id.execSeqNum)
586
587        if full_key in self.insts:
588            return self.insts[full_key]
589        elif macroop_key in self.insts:
590            return self.insts[macroop_key]
591        else:
592            return None
593
594    def add_line(self, line):
595        """Add a MinorLine line to the model"""
596        self.lines[line.id.lineSeqNum] = line
597
598    def add_unit_event(self, event):
599        """Add a single event to the model.  This must be an event at a
600        time >= the current maximum time"""
601        if event.unit in self.unitEvents:
602            events = self.unitEvents[event.unit]
603            if len(events) > 0 and events[len(events)-1].time > event.time:
604                print "Bad event ordering"
605            events.append(event)
606        self.numEvents += 1
607        self.lastTime = max(self.lastTime, event.time)
608
609    def extract_times(self):
610        """Extract a list of all the times from the seen events.  Call after
611        reading events to give a safe index list to use for time indices"""
612        times = {}
613        for unitEvents in self.unitEvents.itervalues():
614            for event in unitEvents:
615                times[event.time] = 1
616        self.times = times.keys()
617        self.times.sort()
618
619    def find_line(self, id):
620        """Find a line by id"""
621        key = id.lineSeqNum
622        return self.lines.get(key, None)
623
624    def find_event_bisection(self, unit, time, events,
625        lower_index, upper_index):
626        """Find an event by binary search on time indices"""
627        while lower_index <= upper_index:
628            pivot = (upper_index + lower_index) / 2
629            pivotEvent = events[pivot]
630            event_equal = (pivotEvent.time == time or
631                (pivotEvent.time < time and
632                    (pivot == len(events) - 1 or
633                        events[pivot + 1].time > time)))
634
635            if event_equal:
636                return pivotEvent
637            elif time > pivotEvent.time:
638                if pivot == upper_index:
639                    return None
640                else:
641                    lower_index = pivot + 1
642            elif time < pivotEvent.time:
643                if pivot == lower_index:
644                    return None
645                else:
646                    upper_index = pivot - 1
647            else:
648                return None
649        return None
650
651    def find_unit_event_by_time(self, unit, time):
652        """Find the last event for the given unit at time <= time"""
653        if unit in self.unitEvents:
654            events = self.unitEvents[unit]
655            ret = self.find_event_bisection(unit, time, events,
656                0, len(events)-1)
657
658            return ret
659        else:
660            return None
661
662    def find_time_index(self, time):
663        """Find a time index close to the given time (where
664        times[return] <= time and times[return+1] > time"""
665        ret = 0
666        lastIndex = len(self.times) - 1
667        while ret < lastIndex and self.times[ret + 1] <= time:
668            ret += 1
669        return ret
670
671    def add_minor_inst(self, rest):
672        """Parse and add a MinorInst line to the model"""
673        pairs = parse.parse_pairs(rest)
674        other_pairs = dict(pairs)
675
676        id = Id().from_string(pairs['id'])
677        del other_pairs['id']
678
679        addr = int(pairs['addr'], 0)
680        del other_pairs['addr']
681
682        if 'inst' in other_pairs:
683            del other_pairs['inst']
684
685            # Collapse unnecessary spaces in disassembly
686            disassembly = re.sub('  *', ' ',
687                re.sub('^ *', '', pairs['inst']))
688
689            inst = Inst(id, disassembly, addr, other_pairs)
690            self.add_inst(inst)
691        elif 'fault' in other_pairs:
692            del other_pairs['fault']
693
694            inst = InstFault(id, pairs['fault'], addr, other_pairs)
695
696            self.add_inst(inst)
697
698    def add_minor_line(self, rest):
699        """Parse and add a MinorLine line to the model"""
700        pairs = parse.parse_pairs(rest)
701        other_pairs = dict(pairs)
702
703        id = Id().from_string(pairs['id'])
704        del other_pairs['id']
705
706        vaddr = int(pairs['vaddr'], 0)
707        del other_pairs['vaddr']
708
709        if 'paddr' in other_pairs:
710            del other_pairs['paddr']
711            del other_pairs['size']
712            paddr = int(pairs['paddr'], 0)
713            size = int(pairs['size'], 0)
714
715            self.add_line(Line(id,
716                vaddr, paddr, size, other_pairs))
717        elif 'fault' in other_pairs:
718            del other_pairs['fault']
719
720            self.add_line(LineFault(id, pairs['fault'], vaddr, other_pairs))
721
722    def load_events(self, file, startTime=0, endTime=None):
723        """Load an event file and add everything to this model"""
724        def update_comments(comments, time):
725            # Add a list of comments to an existing event, if there is one at
726            #   the given time, or create a new, correctly-timed, event from
727            #   the last event and attach the comments to that
728            for commentUnit, commentRest in comments:
729                event = self.find_unit_event_by_time(commentUnit, time)
730                # Find an event to which this comment can be attached
731                if event is None:
732                    # No older event, make a new empty one
733                    event = BlobEvent(commentUnit, time, {})
734                    self.add_unit_event(event)
735                elif event.time != time:
736                    # Copy the old event and make a new one with the right
737                    #   time and comment
738                    newEvent = BlobEvent(commentUnit, time, event.pairs)
739                    newEvent.visuals = dict(event.visuals)
740                    event = newEvent
741                    self.add_unit_event(event)
742                event.comments.append(commentRest)
743
744        self.clear_events()
745
746        # A negative time will *always* be different from an event time
747        time = -1
748        time_events = {}
749        last_time_lines = {}
750        minor_trace_line_count = 0
751        comments = []
752
753        default_colour = [[colours.unknownColour]]
754        next_progress_print_event_count = 1000
755
756        if not os.access(file, os.R_OK):
757            print 'Can\'t open file', file
758            exit(1)
759        else:
760            print 'Opening file', file
761
762        f = open(file)
763
764        start_wall_time = wall_time()
765
766        # Skip leading events
767        still_skipping = True
768        l = f.readline()
769        while l and still_skipping:
770            match = re.match('^\s*(\d+):', l)
771            if match is not None:
772                event_time = match.groups()
773                if int(event_time[0]) >= startTime:
774                    still_skipping = False
775                else:
776                    l = f.readline()
777            else:
778                l = f.readline()
779
780        match_line_re = re.compile(
781            '^\s*(\d+):\s*([\w\.]+):\s*(Minor\w+:)?\s*(.*)$')
782
783        # Parse each line of the events file, accumulating comments to be
784        #   attached to MinorTrace events when the time changes
785        reached_end_time = False
786        while not reached_end_time and l:
787            match = match_line_re.match(l)
788            if match is not None:
789                event_time, unit, line_type, rest = match.groups()
790                event_time = int(event_time)
791
792                unit = re.sub('^' + self.unitNamePrefix + '\.?(.*)$',
793                    '\\1', unit)
794
795                # When the time changes, resolve comments
796                if event_time != time:
797                    if self.numEvents > next_progress_print_event_count:
798                        print ('Parsed to time: %d' % event_time)
799                        next_progress_print_event_count = (
800                            self.numEvents + 1000)
801                    update_comments(comments, time)
802                    comments = []
803                    time = event_time
804
805                if line_type is None:
806                    # Treat this line as just a 'comment'
807                    comments.append((unit, rest))
808                elif line_type == 'MinorTrace:':
809                    minor_trace_line_count += 1
810
811                    # Only insert this event if it's not the same as
812                    #   the last event we saw for this unit
813                    if last_time_lines.get(unit, None) != rest:
814                        event = BlobEvent(unit, event_time, {})
815                        pairs = parse.parse_pairs(rest)
816                        event.pairs = pairs
817
818                        # Try to decode the colour data for this event
819                        blobs = self.unitNameToBlobs.get(unit, [])
820                        for blob in blobs:
821                            if blob.visualDecoder is not None:
822                                event.visuals[blob.picChar] = (
823                                    blob.visualDecoder(pairs))
824
825                        self.add_unit_event(event)
826                        last_time_lines[unit] = rest
827                elif line_type == 'MinorInst:':
828                    self.add_minor_inst(rest)
829                elif line_type == 'MinorLine:':
830                    self.add_minor_line(rest)
831
832            if endTime is not None and time > endTime:
833                reached_end_time = True
834
835            l = f.readline()
836
837        update_comments(comments, time)
838        self.extract_times()
839        f.close()
840
841        end_wall_time = wall_time()
842
843        print 'Total events:', minor_trace_line_count, 'unique events:', \
844            self.numEvents
845        print 'Time to parse:', end_wall_time - start_wall_time
846
847    def add_blob_picture(self, offset, pic, nameDict):
848        """Add a parsed ASCII-art pipeline markup to the model"""
849        pic_width = 0
850        for line in pic:
851            pic_width = max(pic_width, len(line))
852        pic_height = len(pic)
853
854        # Number of horizontal characters per 'pixel'.  Should be 2
855        charsPerPixel = 2
856
857        # Clean up pic_width to a multiple of charsPerPixel
858        pic_width = (pic_width + charsPerPixel - 1) // 2
859
860        self.picSize = Point(pic_width, pic_height)
861
862        def pic_at(point):
863            """Return the char pair at the given point.
864            Returns None for characters off the picture"""
865            x, y = point.to_pair()
866            x *= 2
867            if y >= len(pic) or x >= len(pic[y]):
868                return None
869            else:
870                return pic[y][x:x + charsPerPixel]
871
872        def clear_pic_at(point):
873            """Clear the chars at point so we don't trip over them again"""
874            line = pic[point.y]
875            x = point.x * charsPerPixel
876            pic[point.y] = line[0:x] + (' ' * charsPerPixel) + \
877                line[x + charsPerPixel:]
878
879        def skip_same_char(start, increment):
880            """Skip characters which match pic_at(start)"""
881            char = pic_at(start)
882            hunt = start
883            while pic_at(hunt) == char:
884                hunt += increment
885            return hunt
886
887        def find_size(start):
888            """Find the size of a rectangle with top left hand corner at
889            start consisting of (at least) a -. shaped corner describing
890            the top right corner of a rectangle of the same char"""
891            char = pic_at(start)
892            hunt_x = skip_same_char(start, Point(1,0))
893            hunt_y = skip_same_char(start, Point(0,1))
894            off_bottom_right = (hunt_x * Point(1,0)) + (hunt_y * Point(0,1))
895            return off_bottom_right - start
896
897        def point_return(point):
898            """Carriage return, line feed"""
899            return Point(0, point.y + 1)
900
901        def find_arrow(start):
902            """Find a simple 1-char wide arrow"""
903
904            def body(endChar, contChar, direc):
905                arrow_point = start
906                arrow_point += Point(0, 1)
907                clear_pic_at(start)
908                while pic_at(arrow_point) == contChar:
909                    clear_pic_at(arrow_point)
910                    arrow_point += Point(0, 1)
911
912                if pic_at(arrow_point) == endChar:
913                    clear_pic_at(arrow_point)
914                    self.add_blob(blobs.Arrow('_', start + offset,
915                        direc = direc,
916                        size = (Point(1, 1) + arrow_point - start)))
917                else:
918                    print 'Bad arrow', start
919
920            char = pic_at(start)
921            if char == '-\\':
922                body('-/', ' :', 'right')
923            elif char == '/-':
924                body('\\-', ': ', 'left')
925
926        blank_chars = ['  ', ' :', ': ']
927
928        # Traverse the picture left to right, top to bottom to find blobs
929        seen_dict = {}
930        point = Point(0,0)
931        while pic_at(point) is not None:
932            while pic_at(point) is not None:
933                char = pic_at(point)
934                if char == '->':
935                    self.add_blob(blobs.Arrow('_', point + offset,
936                        direc = 'right'))
937                elif char == '<-':
938                    self.add_blob(blobs.Arrow('_', point + offset,
939                        direc = 'left'))
940                elif char == '-\\' or char == '/-':
941                    find_arrow(point)
942                elif char in blank_chars:
943                    pass
944                else:
945                    if char not in seen_dict:
946                        size = find_size(point)
947                        topLeft = point + offset
948                        if char not in nameDict:
949                            # Unnamed blobs
950                            self.add_blob(blobs.Block(char,
951                                nameDict.get(char, '_'),
952                                topLeft, size = size))
953                        else:
954                            # Named blobs, set visual info.
955                            blob = nameDict[char]
956                            blob.size = size
957                            blob.topLeft = topLeft
958                            self.add_blob(blob)
959                    seen_dict[char] = True
960                point = skip_same_char(point, Point(1,0))
961            point = point_return(point)
962
963    def load_picture(self, filename):
964        """Load a picture file into the model"""
965        def parse_blob_description(char, unit, macros, pairsList):
966            # Parse the name value pairs in a blob-describing line
967            def expand_macros(pairs, newPairs):
968                # Recursively expand macros
969                for name, value in newPairs:
970                    if name in macros:
971                        expand_macros(pairs, macros[name])
972                    else:
973                        pairs[name] = value
974                return pairs
975
976            pairs = expand_macros({}, pairsList)
977
978            ret = None
979
980            typ = pairs.get('type', 'block')
981            colour = colours.name_to_colour(pairs.get('colour', 'black'))
982
983            if typ == 'key':
984                ret = blobs.Key(char, unit, Point(0,0), colour)
985            elif typ == 'block':
986                ret = blobs.Block(char, unit, Point(0,0), colour)
987            else:
988                print "Bad picture blog type:", typ
989
990            if 'hideId' in pairs:
991                hide = pairs['hideId']
992                ret.dataSelect.ids -= set(hide)
993
994            if typ == 'block':
995                ret.displayName = pairs.get('name', unit)
996                ret.nameLoc = pairs.get('nameLoc', 'top')
997                ret.shape = pairs.get('shape', 'box')
998                ret.stripDir = pairs.get('stripDir', 'horiz')
999                ret.stripOrd = pairs.get('stripOrd', 'LR')
1000                ret.blankStrips = int(pairs.get('blankStrips', '0'))
1001                ret.shorten = int(pairs.get('shorten', '0'))
1002
1003                if 'decoder' in pairs:
1004                    decoderName = pairs['decoder']
1005                    dataElement = pairs.get('dataElement', decoderName)
1006
1007                    decoder = find_colour_decoder(ret.blankStrips,
1008                        decoderName, dataElement, pairs)
1009                    if decoder is not None:
1010                        ret.visualDecoder = decoder
1011                    else:
1012                        print 'Bad visualDecoder requested:', decoderName
1013
1014                if 'border' in pairs:
1015                    border = pairs['border']
1016                    if border == 'thin':
1017                        ret.border = 0.2
1018                    elif border == 'mid':
1019                        ret.border = 0.5
1020                    else:
1021                        ret.border = 1.0
1022            elif typ == 'key':
1023                ret.colours = pairs.get('colours', ret.colours)
1024
1025            return ret
1026
1027        def line_is_comment(line):
1028            """Returns true if a line starts with #, returns False
1029            for lines which are None"""
1030            return line is not None \
1031                and re.match('^\s*#', line) is not None
1032
1033        def get_line(f):
1034            """Get a line from file f extending that line if it ends in
1035            '\' and dropping lines that start with '#'s"""
1036            ret = f.readline()
1037
1038            # Discard comment lines
1039            while line_is_comment(ret):
1040                ret = f.readline()
1041
1042            if ret is not None:
1043                extend_match = re.match('^(.*)\\\\$', ret)
1044
1045                while extend_match is not None:
1046                    new_line = f.readline()
1047
1048                    if new_line is not None and not line_is_comment(new_line):
1049                        line_wo_backslash, = extend_match.groups()
1050                        ret = line_wo_backslash + new_line
1051                        extend_match = re.match('^(.*)\\\\$', ret)
1052                    else:
1053                        extend_match = None
1054
1055            return ret
1056
1057        # Macros are recursively expanded into name=value pairs
1058        macros = {}
1059
1060        if not os.access(filename, os.R_OK):
1061            print 'Can\'t open file', filename
1062            exit(1)
1063        else:
1064            print 'Opening file', filename
1065
1066        f = open(filename)
1067        l = get_line(f)
1068        picture = []
1069        blob_char_dict = {}
1070
1071        self.unitEvents = {}
1072        self.clear_events()
1073
1074        # Actually parse the file
1075        in_picture = False
1076        while l:
1077            l = parse.remove_trailing_ws(l)
1078            l = re.sub('#.*', '', l)
1079
1080            if re.match("^\s*$", l) is not None:
1081                pass
1082            elif l == '<<<':
1083                in_picture = True
1084            elif l == '>>>':
1085                in_picture = False
1086            elif in_picture:
1087                picture.append(re.sub('\s*$', '', l))
1088            else:
1089                line_match = re.match(
1090                    '^([a-zA-Z0-9][a-zA-Z0-9]):\s+([\w.]+)\s*(.*)', l)
1091                macro_match = re.match('macro\s+(\w+):(.*)', l)
1092
1093                if macro_match is not None:
1094                    name, defn = macro_match.groups()
1095                    macros[name] = parse.parse_pairs_list(defn)
1096                elif line_match is not None:
1097                    char, unit, pairs = line_match.groups()
1098                    blob = parse_blob_description(char, unit, macros,
1099                        parse.parse_pairs_list(pairs))
1100                    blob_char_dict[char] = blob
1101                    # Setup the events structure
1102                    self.unitEvents[unit] = []
1103                else:
1104                    print 'Problem with Blob line:', l
1105
1106            l = get_line(f)
1107
1108        self.blobs = []
1109        self.add_blob_picture(Point(0,1), picture, blob_char_dict)
1110