Merge pull request #155 from rwightman/densenet_update_and_more

DenseNet updates, EvoNorms,  VovNet, activation factory and more. Includes PR #142

Author: rwightman

README.md

@@ -2,6 +2,19 @@
 
 ## What's New
 
+### June 11, 2020
+Bunch of changes:
+* DenseNet models updated with memory efficient addition from torchvision (fixed a bug), blur pooling and deep stem additions
+* VoVNet V1 and V2 models added, 39 V2 variant (ese_vovnet_39b) trained to 79.3 top-1
+* Activation factory added along with new activations:
+   * select act at model creation time for more flexibility in using activations compatible with scripting or tracing (ONNX export)
+   * hard_mish (experimental) added with memory-efficient grad, along with ME hard_swish
+   * context mgr for setting exportable/scriptable/no_jit states
+* Norm + Activation combo layers added with initial trial support in DenseNet and VoVNet along with impl of EvoNorm and InplaceAbn wrapper that fit the interface
+* Torchscript works for all but two of the model types as long as using Pytorch 1.5+, tests added for this
+* Some import cleanup and classifier reset changes, all models will have classifier reset to nn.Identity on reset_classifer(0) call
+* Prep for 0.1.28 pip release
+
 ### May 12, 2020
 * Add ResNeSt models (code adapted from https://github.com/zhanghang1989/ResNeSt, paper https://arxiv.org/abs/2004.08955))
 

results/results-imagenet-a.csv

@@ -14,7 +14,9 @@ tf_efficientnet_b8,29.3733,70.6267,57.0667,42.9333,87.41,672,0.954,bicubic
 ig_resnext101_32x8d,28.7067,71.2933,52.32,47.68,88.79,224,0.875,bilinear
 swsl_resnext101_32x16d,27.9467,72.0533,52.32,47.68,194.03,224,0.875,bilinear
 tf_efficientnet_b7_ap,27.8133,72.1867,54.7733,45.2267,66.35,600,0.949,bicubic
+resnest269e,27.6133,72.3867,53.1067,46.8933,110.93,416,0.875,bilinear
 tresnet_xl_448,26.88,73.12,51.0933,48.9067,78.44,448,0.875,bilinear
+resnest200e,26.4267,73.5733,51.9333,48.0667,70.2,320,0.875,bilinear
 swsl_resnext101_32x4d,25.3467,74.6533,49.6267,50.3733,44.18,224,0.875,bilinear
 tf_efficientnet_b7,25.2533,74.7467,51.6667,48.3333,66.35,600,0.949,bicubic
 tresnet_l_448,24.5733,75.4267,48.6,51.4,55.99,448,0.875,bilinear
@@ -26,6 +28,7 @@ tf_efficientnet_b3_ns,19.4133,80.5867,44.6267,55.3733,12.23,300,0.904,bicubic
 swsl_resnext50_32x4d,18.0667,81.9333,41.8667,58.1333,25.03,224,0.875,bilinear
 ssl_resnext101_32x16d,17.2133,82.7867,39.9467,60.0533,194.03,224,0.875,bilinear
 tf_efficientnet_b5,17.0667,82.9333,41.9067,58.0933,30.39,456,0.934,bicubic
+resnest101e,16.4933,83.5067,40.7467,59.2533,48.28,256,0.875,bilinear
 swsl_resnet50,15.9867,84.0133,38.8533,61.1467,25.56,224,0.875,bilinear
 ssl_resnext101_32x8d,15.12,84.88,37.72,62.28,88.79,224,0.875,bilinear
 tf_efficientnet_b4_ap,13.68,86.32,35.92,64.08,19.34,380,0.922,bicubic
@@ -36,14 +39,16 @@ nasnetalarge,12.5733,87.4267,33.4133,66.5867,88.75,331,0.875,bicubic
 ssl_resnext101_32x4d,12.12,87.88,31.8933,68.1067,44.18,224,0.875,bilinear
 tf_efficientnet_b2_ns,11.7867,88.2133,32.96,67.04,9.11,260,0.89,bicubic
 gluon_senet154,9.9067,90.0933,26.4533,73.5467,115.09,224,0.875,bicubic
+resnest50d_4s2x40d,9.7867,90.2133,29.1467,70.8533,30.42,224,0.875,bicubic
 ssl_resnext50_32x4d,9.6667,90.3333,28.4267,71.5733,25.03,224,0.875,bilinear
 senet154,9.4533,90.5467,26.44,73.56,115.09,224,0.875,bilinear
 tresnet_xl,9.3067,90.6933,28.4133,71.5867,78.44,224,0.875,bilinear
-efficientnet_b3a,9.2667,90.7333,28.4267,71.5733,12.23,320,1,bicubic
+efficientnet_b3a,9.2667,90.7333,28.4267,71.5733,12.23,320,1.0,bicubic
 efficientnet_b3,8.9467,91.0533,28.2133,71.7867,12.23,300,0.904,bicubic
 inception_v4,8.92,91.08,24.7067,75.2933,42.68,299,0.875,bicubic
 gluon_seresnext101_64x4d,8.8667,91.1333,27.32,72.68,88.23,224,0.875,bicubic
 tf_efficientnet_b1_ns,8.6133,91.3867,27.28,72.72,7.79,240,0.882,bicubic
+resnest50d_1s4x24d,8.52,91.48,26.7867,73.2133,25.68,224,0.875,bicubic
 ecaresnet50d,8.5067,91.4933,26.2667,73.7333,25.58,224,0.875,bicubic
 gluon_xception65,8.4667,91.5333,25.1333,74.8667,39.92,299,0.875,bicubic
 gluon_resnet152_v1d,8.4133,91.5867,23.4533,76.5467,60.21,224,0.875,bicubic
@@ -55,23 +60,30 @@ ens_adv_inception_resnet_v2,7.9867,92.0133,23.8267,76.1733,55.84,299,0.8975,bicu
 tf_efficientnet_lite4,7.9333,92.0667,25.56,74.44,13.01,380,0.92,bilinear
 tresnet_l,7.88,92.12,25.1867,74.8133,55.99,224,0.875,bilinear
 gluon_resnet152_v1s,7.8667,92.1333,23.1733,76.8267,60.32,224,0.875,bicubic
+resnest50d,7.7467,92.2533,25.2933,74.7067,27.48,224,0.875,bilinear
 gluon_resnext101_64x4d,7.7067,92.2933,23.24,76.76,83.46,224,0.875,bicubic
 skresnext50_32x4d,7.08,92.92,23.0267,76.9733,27.48,224,0.875,bicubic
-ssl_resnet50,7,93,23.92,76.08,25.56,224,0.875,bilinear
+ssl_resnet50,7.0,93.0,23.92,76.08,25.56,224,0.875,bilinear
+regnety_320,6.92,93.08,23.04,76.96,145.05,224,0.875,bicubic
 ecaresnet101d_pruned,6.8,93.2,24.2,75.8,24.88,224,0.875,bicubic
 ecaresnetlight,6.76,93.24,22.56,77.44,30.16,224,0.875,bicubic
-efficientnet_b2a,6.76,93.24,23.4933,76.5067,9.11,288,1,bicubic
+efficientnet_b2a,6.76,93.24,23.4933,76.5067,9.11,288,1.0,bicubic
 seresnext101_32x4d,6.4133,93.5867,21.52,78.48,48.96,224,0.875,bilinear
 efficientnet_b2,6.0933,93.9067,21.9333,78.0667,9.11,260,0.875,bicubic
 gluon_resnext101_32x4d,6.04,93.96,21.1333,78.8667,44.18,224,0.875,bicubic
+regnetx_320,5.9867,94.0133,19.88,80.12,107.81,224,0.875,bicubic
+ese_vovnet39b,5.9733,94.0267,21.2933,78.7067,24.57,224,0.875,bicubic
 gluon_resnet101_v1d,5.92,94.08,19.9467,80.0533,44.57,224,0.875,bicubic
 gluon_seresnext50_32x4d,5.7867,94.2133,21.4267,78.5733,27.56,224,0.875,bicubic
 efficientnet_b3_pruned,5.7333,94.2667,21.36,78.64,9.86,300,0.904,bicubic
+regnety_160,5.64,94.36,19.3467,80.6533,83.59,224,0.875,bicubic
 gluon_inception_v3,5.5067,94.4933,19.9467,80.0533,23.83,299,0.875,bicubic
 mixnet_xl,5.48,94.52,21.0933,78.9067,11.9,224,0.875,bicubic
 tresnet_m,5.44,94.56,19.96,80.04,31.39,224,0.875,bilinear
+regnety_120,5.4133,94.5867,19.8533,80.1467,51.82,224,0.875,bicubic
 gluon_resnet101_v1s,5.28,94.72,19.5467,80.4533,44.67,224,0.875,bicubic
 hrnet_w64,5.1333,94.8667,19.4533,80.5467,128.06,224,0.875,bilinear
+regnety_080,5.0,95.0,18.6,81.4,39.18,224,0.875,bicubic
 efficientnet_b2_pruned,4.9467,95.0533,19.3467,80.6533,8.31,260,0.89,bicubic
 dpn107,4.88,95.12,17.6133,82.3867,86.92,224,0.875,bicubic
 gluon_resnet152_v1c,4.8667,95.1333,17.7733,82.2267,60.21,224,0.875,bicubic
@@ -84,38 +96,45 @@ gluon_resnet152_v1b,4.5867,95.4133,16.5333,83.4667,60.19,224,0.875,bicubic
 ecaresnet50d_pruned,4.5467,95.4533,18.5467,81.4533,19.94,224,0.875,bicubic
 dpn92,4.4933,95.5067,18.2,81.8,37.67,224,0.875,bicubic
 hrnet_w44,4.4933,95.5067,17.3467,82.6533,67.06,224,0.875,bilinear
+regnetx_160,4.3733,95.6267,17.0933,82.9067,54.28,224,0.875,bicubic
 resnext50d_32x4d,4.3467,95.6533,17.7733,82.2267,25.05,224,0.875,bicubic
 xception,4.3467,95.6533,16.76,83.24,22.86,299,0.8975,bicubic
 seresnext50_32x4d,4.28,95.72,17.8133,82.1867,27.56,224,0.875,bilinear
 resnext50_32x4d,4.2533,95.7467,18.3867,81.6133,25.03,224,0.875,bicubic
 tf_efficientnet_cc_b1_8e,4.24,95.76,15.9467,84.0533,39.72,240,0.882,bicubic
+regnety_064,4.2267,95.7733,17.1867,82.8133,30.58,224,0.875,bicubic
 tf_efficientnet_el,4.2267,95.7733,18.1733,81.8267,10.59,300,0.904,bicubic
-inception_v3,4.2,95.8,16.2933,83.7067,27.16,299,0.875,bicubic
+inception_v3,4.1867,95.8133,16.2933,83.7067,23.83,299,0.875,bicubic
 tf_efficientnet_b2_ap,4.1733,95.8267,18.32,81.68,9.11,260,0.89,bicubic
 seresnet152,4.1467,95.8533,15.8933,84.1067,66.82,224,0.875,bilinear
 resnext101_32x8d,4.1333,95.8667,16.9867,83.0133,88.79,224,0.875,bilinear
 tf_efficientnet_b0_ns,4.1333,95.8667,17.68,82.32,5.29,224,0.875,bicubic
 dpn98,4.08,95.92,15.9467,84.0533,61.57,224,0.875,bicubic
-res2net101_26w_4s,4,96,14.8267,85.1733,45.21,224,0.875,bilinear
+res2net101_26w_4s,4.0,96.0,14.8267,85.1733,45.21,224,0.875,bilinear
 efficientnet_b1,3.9733,96.0267,15.76,84.24,7.79,240,0.875,bicubic
 tf_efficientnet_lite3,3.9333,96.0667,16.52,83.48,8.2,300,0.904,bilinear
 tf_efficientnet_b2,3.7733,96.2267,16.6133,83.3867,9.11,260,0.89,bicubic
+regnety_040,3.7467,96.2533,16.4,83.6,20.65,224,0.875,bicubic
 hrnet_w30,3.68,96.32,15.5733,84.4267,37.71,224,0.875,bilinear
 hrnet_w32,3.6533,96.3467,14.7867,85.2133,41.23,224,0.875,bilinear
 hrnet_w40,3.6533,96.3467,15.44,84.56,57.56,224,0.875,bilinear
+regnetx_120,3.6267,96.3733,15.9733,84.0267,46.11,224,0.875,bicubic
 seresnext26t_32x4d,3.6133,96.3867,15.8933,84.1067,16.82,224,0.875,bicubic
 tf_efficientnet_b1_ap,3.5467,96.4533,15.0667,84.9333,7.79,240,0.882,bicubic
 seresnext26tn_32x4d,3.5067,96.4933,15.76,84.24,16.81,224,0.875,bicubic
+resnest26d,3.4933,96.5067,15.6667,84.3333,17.07,224,0.875,bilinear
 dla169,3.4667,96.5333,15.3333,84.6667,53.99,224,0.875,bilinear
 gluon_resnext50_32x4d,3.4533,96.5467,16.12,83.88,25.03,224,0.875,bicubic
 mixnet_l,3.44,96.56,15.3067,84.6933,7.33,224,0.875,bicubic
 seresnext26d_32x4d,3.4,96.6,16.16,83.84,16.81,224,0.875,bicubic
-resnetblur50,3.3333,96.6667,15.5867,84.4133,25.56,224,0.875,bicubic
 res2net50_26w_8s,3.3333,96.6667,14.04,85.96,48.4,224,0.875,bilinear
+resnetblur50,3.3333,96.6667,15.5867,84.4133,25.56,224,0.875,bicubic
 dla102x,3.3067,96.6933,15.12,84.88,26.77,224,0.875,bilinear
 gluon_resnet101_v1c,3.3067,96.6933,14.12,85.88,44.57,224,0.875,bicubic
 seresnet101,3.2533,96.7467,15.4533,84.5467,49.33,224,0.875,bilinear
+densenetblur121d,3.0667,96.9333,14.28,85.72,8.0,224,0.875,bicubic
 dla60_res2next,3.04,96.96,14.4533,85.5467,17.33,224,0.875,bilinear
+regnety_032,3.0267,96.9733,14.24,85.76,19.44,224,0.875,bicubic
 gluon_resnet50_v1d,3.0133,96.9867,14.6267,85.3733,25.58,224,0.875,bicubic
 wide_resnet101_2,2.96,97.04,13.9467,86.0533,126.89,224,0.875,bilinear
 efficientnet_b1_pruned,2.9333,97.0667,14.4133,85.5867,6.33,240,0.882,bicubic
@@ -124,6 +143,7 @@ tf_efficientnet_b1,2.8667,97.1333,13.5067,86.4933,7.79,240,0.882,bicubic
 res2net50_26w_6s,2.84,97.16,12.6,87.4,37.05,224,0.875,bilinear
 efficientnet_b0,2.8133,97.1867,13.9067,86.0933,5.29,224,0.875,bicubic
 tf_mixnet_l,2.8133,97.1867,13.04,86.96,7.33,224,0.875,bicubic
+regnetx_064,2.7867,97.2133,13.88,86.12,26.21,224,0.875,bicubic
 dpn68b,2.7067,97.2933,12.64,87.36,12.61,224,0.875,bicubic
 selecsls60b,2.6933,97.3067,13.1733,86.8267,32.77,224,0.875,bicubic
 tf_efficientnet_cc_b0_8e,2.68,97.32,12.7733,87.2267,24.01,224,0.875,bicubic
@@ -134,24 +154,30 @@ mixnet_m,2.5467,97.4533,12.4267,87.5733,5.01,224,0.875,bicubic
 skresnet34,2.52,97.48,12.7733,87.2267,22.28,224,0.875,bicubic
 efficientnet_es,2.3733,97.6267,13.88,86.12,5.44,224,0.875,bicubic
 resnet152,2.36,97.64,12.2,87.8,60.19,224,0.875,bilinear
+regnetx_080,2.3467,97.6533,12.6933,87.3067,39.57,224,0.875,bicubic
 swsl_resnet18,2.3333,97.6667,11.2133,88.7867,11.69,224,0.875,bilinear
 wide_resnet50_2,2.32,97.68,11.8,88.2,68.88,224,0.875,bilinear
 seresnext26_32x4d,2.2933,97.7067,12.44,87.56,16.79,224,0.875,bicubic
 hrnet_w18,2.2667,97.7333,11.8533,88.1467,21.3,224,0.875,bilinear
 dla102,2.2533,97.7467,12.12,87.88,33.73,224,0.875,bilinear
 resnet50,2.2267,97.7733,11.3333,88.6667,25.56,224,0.875,bicubic
+regnety_016,2.1733,97.8267,11.44,88.56,11.2,224,0.875,bicubic
+regnetx_040,2.16,97.84,11.8,88.2,22.12,224,0.875,bicubic
+resnest14d,2.1467,97.8533,10.4,89.6,10.61,224,0.875,bilinear
 selecsls60,2.08,97.92,12.84,87.16,30.67,224,0.875,bicubic
 tf_efficientnet_cc_b0_4e,2.08,97.92,10.9733,89.0267,13.31,224,0.875,bicubic
 res2next50,2.0667,97.9333,11.4533,88.5467,24.67,224,0.875,bilinear
 seresnet50,2.0667,97.9333,12.2667,87.7333,28.09,224,0.875,bilinear
 densenet161,1.9733,98.0267,10.5867,89.4133,28.68,224,0.875,bicubic
 tf_efficientnet_b0_ap,1.96,98.04,10.8,89.2,5.29,224,0.875,bicubic
+regnetx_032,1.92,98.08,10.9467,89.0533,15.3,224,0.875,bicubic
 tf_efficientnet_em,1.8133,98.1867,11.6267,88.3733,6.9,240,0.882,bicubic
 tf_mixnet_m,1.8133,98.1867,10.5467,89.4533,5.01,224,0.875,bicubic
 tf_efficientnet_lite2,1.8,98.2,11.1467,88.8533,6.09,260,0.89,bicubic
 res2net50_14w_8s,1.7867,98.2133,10.3467,89.6533,25.06,224,0.875,bilinear
 res2net50_26w_4s,1.7733,98.2267,10.44,89.56,25.7,224,0.875,bilinear
 mobilenetv3_large_100,1.76,98.24,10.2933,89.7067,5.48,224,0.875,bicubic
+densenet121,1.7333,98.2667,10.8533,89.1467,7.98,224,0.875,bicubic
 tf_efficientnet_b0,1.6933,98.3067,9.7333,90.2667,5.29,224,0.875,bicubic
 tv_resnext50_32x4d,1.68,98.32,10.6,89.4,25.03,224,0.875,bilinear
 mobilenetv3_rw,1.6667,98.3333,10.7333,89.2667,5.48,224,0.875,bicubic
@@ -163,6 +189,7 @@ gluon_resnet50_v1c,1.5467,98.4533,10.6133,89.3867,25.58,224,0.875,bicubic
 semnasnet_100,1.5467,98.4533,9.32,90.68,3.89,224,0.875,bicubic
 selecsls42b,1.4667,98.5333,10.44,89.56,32.46,224,0.875,bicubic
 tf_efficientnet_lite1,1.4533,98.5467,9.7067,90.2933,5.42,240,0.882,bicubic
+regnety_008,1.4267,98.5733,8.9467,91.0533,6.26,224,0.875,bicubic
 ssl_resnet18,1.3867,98.6133,8.16,91.84,11.69,224,0.875,bilinear
 dla60,1.3467,98.6533,9.4667,90.5333,22.33,224,0.875,bilinear
 dpn68,1.3467,98.6533,8.8133,91.1867,12.61,224,0.875,bicubic
@@ -178,19 +205,26 @@ seresnet34,1.12,98.88,7.4,92.6,21.96,224,0.875,bilinear
 tf_efficientnet_es,1.12,98.88,8.6,91.4,5.44,224,0.875,bicubic
 spnasnet_100,1.1067,98.8933,8.2533,91.7467,4.42,224,0.875,bilinear
 tf_efficientnet_lite0,1.1067,98.8933,7.4933,92.5067,4.65,224,0.875,bicubic
+regnetx_016,1.0933,98.9067,8.6267,91.3733,9.19,224,0.875,bicubic
 dla34,1.08,98.92,7.6933,92.3067,15.78,224,0.875,bilinear
+regnety_006,1.0533,98.9467,8.4,91.6,6.06,224,0.875,bicubic
+regnety_004,1.0133,98.9867,7.3333,92.6667,4.34,224,0.875,bicubic
 resnet34,0.9867,99.0133,7.5333,92.4667,21.8,224,0.875,bilinear
 mobilenetv2_110d,0.9333,99.0667,8.1067,91.8933,4.52,224,0.875,bicubic
 gluon_resnet34_v1b,0.8933,99.1067,6.6,93.4,21.8,224,0.875,bicubic
 hrnet_w18_small_v2,0.8933,99.1067,7.3867,92.6133,15.6,224,0.875,bilinear
+regnetx_008,0.8933,99.1067,6.9067,93.0933,7.26,224,0.875,bicubic
 skresnet18,0.88,99.12,7.3867,92.6133,11.96,224,0.875,bicubic
 mnasnet_100,0.8667,99.1333,7.8667,92.1333,4.38,224,0.875,bicubic
 tf_mobilenetv3_large_075,0.8667,99.1333,6.72,93.28,3.99,224,0.875,bilinear
+regnetx_006,0.76,99.24,6.4933,93.5067,6.2,224,0.875,bicubic
 tf_mobilenetv3_small_100,0.7467,99.2533,4.6667,95.3333,2.54,224,0.875,bilinear
 seresnet18,0.72,99.28,6.0267,93.9733,11.78,224,0.875,bicubic
-densenet121,0.68,99.32,6.9067,93.0933,7.98,224,0.875,bicubic
+regnetx_004,0.6933,99.3067,5.5067,94.4933,5.16,224,0.875,bicubic
+tv_densenet121,0.68,99.32,6.9067,93.0933,7.98,224,0.875,bicubic
+regnety_002,0.6667,99.3333,5.5333,94.4667,3.16,224,0.875,bicubic
 tf_mobilenetv3_small_075,0.6267,99.3733,4.1733,95.8267,2.04,224,0.875,bilinear
-resnet26,0.6,99.4,6.88,93.12,16,224,0.875,bicubic
+resnet26,0.6,99.4,6.88,93.12,16.0,224,0.875,bicubic
 tv_resnet34,0.6,99.4,5.52,94.48,21.8,224,0.875,bilinear
 mobilenetv2_100,0.5333,99.4667,6.1867,93.8133,3.5,224,0.875,bicubic
 dla46_c,0.52,99.48,4.1867,95.8133,1.31,224,0.875,bilinear
@@ -201,4 +235,5 @@ dla46x_c,0.4133,99.5867,4.44,95.56,1.08,224,0.875,bilinear
 gluon_resnet18_v1b,0.3867,99.6133,4.7867,95.2133,11.69,224,0.875,bicubic
 tf_mobilenetv3_small_minimal_100,0.36,99.64,2.8667,97.1333,2.04,224,0.875,bilinear
 resnet18,0.2933,99.7067,4.04,95.96,11.69,224,0.875,bilinear
-tv_resnet50,0,100,2.8933,97.1067,25.56,224,0.875,bilinear
+regnetx_002,0.2267,99.7733,3.9867,96.0133,2.68,224,0.875,bicubic
+tv_resnet50,0.0,100.0,2.8933,97.1067,25.56,224,0.875,bilinear