Update results csv file rank/diff script and small validate script tweak for batch validation

Author: rwightman

results/generate_csv_results.py

@@ -1,57 +1,75 @@
 import numpy as np
 import pandas as pd
 
+
 results = {
-    'results-imagenet.csv': pd.read_csv('results-imagenet.csv'),
-    'results-imagenetv2-matched-frequency.csv': pd.read_csv('results-imagenetv2-matched-frequency.csv'),
-    'results-sketch.csv': pd.read_csv('results-sketch.csv'),
-    'results-imagenet-a.csv': pd.read_csv('results-imagenet-a.csv'),
-    'results-imagenet-r.csv': pd.read_csv('results-imagenet-r.csv'),
-    'results-imagenet-real.csv': pd.read_csv('results-imagenet-real.csv'),
+    'results-imagenet.csv': [
+        'results-imagenet-real.csv',
+        'results-imagenetv2-matched-frequency.csv',
+        'results-sketch.csv'
+    ],
+    'results-imagenet-a-clean.csv': [
+        'results-imagenet-a.csv',
+    ],
+    'results-imagenet-r-clean.csv': [
+        'results-imagenet-r.csv',
+    ],
 }
 
 
-def diff(csv_file):    
-    base_models = results['results-imagenet.csv']['model'].values
-    csv_models  = results[csv_file]['model'].values
+def diff(base_df, test_csv):
+    base_models = base_df['model'].values
+    test_df = pd.read_csv(test_csv)
+    test_models  = test_df['model'].values
 
-    rank_diff = np.zeros_like(csv_models, dtype='object')
-    top1_diff = np.zeros_like(csv_models, dtype='object')
-    top5_diff = np.zeros_like(csv_models, dtype='object')
+    rank_diff = np.zeros_like(test_models, dtype='object')
+    top1_diff = np.zeros_like(test_models, dtype='object')
+    top5_diff = np.zeros_like(test_models, dtype='object')
 
-    for rank, model in enumerate(csv_models):
+    for rank, model in enumerate(test_models):
         if model in base_models:            
-            base_rank = int(np.where(base_models==model)[0])
-            top1_d = results[csv_file]['top1'][rank]-results['results-imagenet.csv']['top1'][base_rank]
-            top5_d = results[csv_file]['top5'][rank]-results['results-imagenet.csv']['top5'][base_rank]
+            base_rank = int(np.where(base_models == model)[0])
+            top1_d = test_df['top1'][rank] - base_df['top1'][base_rank]
+            top5_d = test_df['top5'][rank] - base_df['top5'][base_rank]
 
             # rank_diff
-            if   rank == base_rank: rank_diff[rank] = f'='
-            elif rank >  base_rank: rank_diff[rank] = f'-{rank-base_rank}'
-            else:                   rank_diff[rank] = f'+{base_rank-rank}'
+            if rank == base_rank:
+                rank_diff[rank] = f'0'
+            elif rank > base_rank:
+                rank_diff[rank] = f'-{rank - base_rank}'
+            else:
+                rank_diff[rank] = f'+{base_rank - rank}'
 
             # top1_diff
-            if top1_d >= .0: top1_diff[rank] = f'+{top1_d:.3f}'
-            else           : top1_diff[rank] = f'-{abs(top1_d):.3f}'
+            if top1_d >= .0:
+                top1_diff[rank] = f'+{top1_d:.3f}'
+            else:
+                top1_diff[rank] = f'-{abs(top1_d):.3f}'
 
             # top5_diff
-            if top5_d >= .0: top5_diff[rank] = f'+{top5_d:.3f}'
-            else           : top5_diff[rank] = f'-{abs(top5_d):.3f}'
+            if top5_d >= .0:
+                top5_diff[rank] = f'+{top5_d:.3f}'
+            else:
+                top5_diff[rank] = f'-{abs(top5_d):.3f}'
 
         else: 
-            rank_diff[rank] = 'X'
-            top1_diff[rank] = 'X'
-            top5_diff[rank] = 'X'
-    
-    results[csv_file]['rank_diff'] = rank_diff
-    results[csv_file]['top1_diff'] = top1_diff
-    results[csv_file]['top5_diff'] = top5_diff
-    
-    results[csv_file]['param_count'] = results[csv_file]['param_count'].map('{:,.2f}'.format)
+            rank_diff[rank] = ''
+            top1_diff[rank] = ''
+            top5_diff[rank] = ''
+
+    test_df['top1_diff'] = top1_diff
+    test_df['top5_diff'] = top5_diff
+    test_df['rank_diff'] = rank_diff
 
-    results[csv_file].to_csv(csv_file, index=False, float_format='%.3f')
+    test_df['param_count'] = test_df['param_count'].map('{:,.2f}'.format)
+    test_df.sort_values('top1', ascending=False, inplace=True)
+    test_df.to_csv(test_csv, index=False, float_format='%.3f')
 
 
-for csv_file in results:
-    if csv_file != 'results-imagenet.csv':
-        diff(csv_file)
+for base_results, test_results in results.items():
+    base_df = pd.read_csv(base_results)
+    base_df.sort_values('top1', ascending=False, inplace=True)
+    for test_csv in test_results:
+        diff(base_df, test_csv)
+    base_df['param_count'] = base_df['param_count'].map('{:,.2f}'.format)
+    base_df.to_csv(base_results, index=False, float_format='%.3f')

validate.py

@@ -271,6 +271,7 @@ def main():
                 result = OrderedDict(model=args.model)
                 r = {}
                 while not r and batch_size >= args.num_gpu:
+                    torch.cuda.empty_cache()
                     try:
                         args.batch_size = batch_size
                         print('Validating with batch size: %d' % args.batch_size)
@@ -281,7 +282,6 @@ def main():
                             raise e
                         batch_size = max(batch_size // 2, args.num_gpu)
                         print("Validation failed, reducing batch size by 50%")
-                        torch.cuda.empty_cache()
                 result.update(r)
                 if args.checkpoint:
                     result['checkpoint'] = args.checkpoint