helm/software/matita/bench_summary.py

   1 #!/usr/bin/python
   2 import sys
   3
   4 stats = {}
   5 stats['precise'] = []
   6 stats['imprecise'] = []
   7 stats['total'] = 0
   8 stats['spurious'] = {'tot': 0, 'max': 0, 'names': []}
   9 stats['half-mistakes'] = []
  10 stats['mistakes'] = []
  11 stats['undetected'] = []
  12
  13 for line in open(sys.argv[1]):
  14     if line[0] == '#':
  15         continue
  16     cols = line.split('|')
  17     name = cols[0]
  18     if cols[1] != 'KO':
  19         print >> sys.stderr, "Warning: outcome is %s" % cols[1]
  20         continue
  21     stats['total'] += 1
  22     expected_error = cols[2]
  23     real_errors = cols[3].split(',')
  24     spurious_errors = cols[4].split(',')
  25
  26     if [expected_error] == real_errors:
  27         if expected_error in spurious_errors:
  28             stats['half-mistakes'].append(name)
  29         else:
  30             stats['precise'].append(name)
  31             stats['spurious']['max'] = max(len(spurious_errors), stats['spurious']['max'])
  32             stats['spurious']['tot'] += len(spurious_errors)
  33     elif expected_error in real_errors:
  34         if expected_error in spurious_errors:
  35             stats['half-mistakes'].append(name)
  36         else:
  37             stats['imprecise'].append(name)
  38     else:
  39         if expected_error in spurious_errors:
  40             stats['mistakes'].append(name)
  41         else:
  42             stats['undetected'].append(name)
  43
  44 for field in ['undetected', 'imprecise', 'mistakes', 'half-mistakes']:
  45     print "===================="
  46     print "%s:" % field
  47     for name in stats[field]:
  48         print "  %s" % name
  49 print "\n"
  50 print 'precise:\t%3d' % len(stats['precise'])
  51 print 'imprecise:\t%3d' % len(stats['imprecise'])
  52 print 'half-mistakes:\t%3d' % len(stats['half-mistakes'])
  53 print 'mistakes:\t%3d' % len(stats['mistakes'])
  54 print 'undetected:\t%3d' % len(stats['undetected'])
  55 print 'TOTAL:\t\t%3d' % stats['total'], "\n"
  56
  57 print 'average spurious errors no: ', float(stats['spurious']['tot']) / len(stats['precise'])
  58
  59 #for field, value in stats.iteritems():
  60 #    if field == 'spurious':
  61 #        value['avg'] = float(value['tot']) / stats['precise']
  62 #    print "%s: %s" % (field, value)
  63