helm/software/matita/bench_summary.py

   1 #!/usr/bin/python
   2 import sys
   3
   4 stats = {}
   5 stats['precise'] = []
   6 stats['imprecise'] = []
   7 stats['total'] = 0
   8 stats['spurious'] = {'tot': 0, 'max': 0, 'names': []}
   9 stats['half-mistakes'] = []
  10 stats['mistakes'] = []
  11 stats['undetected'] = []
  12
  13 for line in open(sys.argv[1]):
  14     line = line.rstrip()
  15     if line[0] == '#':
  16         continue
  17     cols = line.split('|')
  18     name = cols[0]
  19     if cols[1] != 'KO':
  20         print "Warning: outcome of %s is %s, skipping it" % (name, cols[1])
  21         continue
  22     stats['total'] += 1
  23     expected_error = cols[2]
  24     real_errors = cols[3].split(',')
  25     spurious_errors = cols[4].split(',')
  26
  27     if [expected_error] == real_errors:
  28         if expected_error in spurious_errors:
  29             stats['half-mistakes'].append(name)
  30         else:
  31             stats['precise'].append(name)
  32             stats['spurious']['max'] = max(len(spurious_errors), stats['spurious']['max'])
  33             stats['spurious']['tot'] += len(spurious_errors)
  34     elif expected_error in real_errors:
  35         if expected_error in spurious_errors:
  36             stats['half-mistakes'].append(name)
  37         else:
  38             stats['imprecise'].append(name)
  39     else:
  40         if expected_error in spurious_errors:
  41             stats['mistakes'].append(name)
  42         else:
  43             stats['undetected'].append(name)
  44
  45 for field in ['undetected', 'imprecise', 'mistakes', 'half-mistakes']:
  46     print "===================="
  47     print "%s:" % field
  48     for name in stats[field]:
  49         print "  %s" % name
  50
  51 def get_stat(field):
  52     global stats
  53     return (stat, float(stat) / stats['total'] * 100)
  54
  55 def format_stat(field):
  56     global stats
  57     if isinstance(stats[field], int):
  58         stat = stats[field]
  59     else:
  60         stat = len(stats[field])
  61     return r'%-15s & %6d & %6.1f\%% \\' % (field, stat,
  62             float(stat) / stats['total'] * 100)
  63
  64 print "\n"
  65 print format_stat('precise')
  66 print format_stat('imprecise')
  67 print format_stat('half-mistakes')
  68 print format_stat('mistakes')
  69 print format_stat('undetected')
  70 print r'\hline'
  71 print format_stat('total')
  72 print r'\\'
  73 print r'%-15s & %6.1f & \\' % ('spurious average',
  74         float(stats['spurious']['tot']) / len(stats['precise']))
  75 print r'%-15s & %6d & \\' % ('spurious max',
  76         float(stats['spurious']['max']))
  77 print r'\hline'
  78