helm/software/matita/bench_summary.py

   1 #!/usr/bin/python
   2 import sys
   3
   4 stats = {}
   5 stats['precise'] = []
   6 stats['imprecise'] = []
   7 stats['total'] = 0
   8 stats['spurious'] = {'tot': 0, 'max': 0, 'names': []}
   9 stats['half-mistakes'] = []
  10 stats['mistakes'] = []
  11 stats['undetected'] = []
  12
  13 for line in open(sys.argv[1]):
  14     if line[0] == '#':
  15         continue
  16     cols = line.split('|')
  17     name = cols[0]
  18     if cols[1] != 'KO':
  19         print "Warning: outcome of %s is %s, skipping it" % (name, cols[1])
  20         continue
  21     stats['total'] += 1
  22     expected_error = cols[2]
  23     real_errors = cols[3].split(',')
  24     spurious_errors = cols[4].split(',')
  25
  26     if [expected_error] == real_errors:
  27         if expected_error in spurious_errors:
  28             stats['half-mistakes'].append(name)
  29         else:
  30             stats['precise'].append(name)
  31             stats['spurious']['max'] = max(len(spurious_errors), stats['spurious']['max'])
  32             stats['spurious']['tot'] += len(spurious_errors)
  33     elif expected_error in real_errors:
  34         if expected_error in spurious_errors:
  35             stats['half-mistakes'].append(name)
  36         else:
  37             stats['imprecise'].append(name)
  38     else:
  39         if expected_error in spurious_errors:
  40             stats['mistakes'].append(name)
  41         else:
  42             stats['undetected'].append(name)
  43
  44 for field in ['undetected', 'imprecise', 'mistakes', 'half-mistakes']:
  45     print "===================="
  46     print "%s:" % field
  47     for name in stats[field]:
  48         print "  %s" % name
  49
  50 def get_stat(field):
  51     global stats
  52     return (stat, float(stat) / stats['total'] * 100)
  53
  54 def format_stat(field):
  55     global stats
  56     if isinstance(stats[field], int):
  57         stat = stats[field]
  58     else:
  59         stat = len(stats[field])
  60     return r'%-15s & %6d & %6.1f\%% \\' % (field, stat,
  61             float(stat) / stats['total'] * 100)
  62
  63 print "\n"
  64 print format_stat('precise')
  65 print format_stat('imprecise')
  66 print format_stat('half-mistakes')
  67 print format_stat('mistakes')
  68 print format_stat('undetected')
  69 print r'\hline'
  70 print format_stat('total')
  71 print r'\\'
  72 print r'%-15s & %6.1f & \\' % ('spurious avg',
  73         float(stats['spurious']['tot']) / len(stats['precise']))
  74 print r'%-15s & %6d & \\' % ('spurious max',
  75         float(stats['spurious']['max']))
  76 print r'\hline'
  77