...
The following uses nodecomm to measure the bandwidth between 2 nodes, with 1-32 processes on each node communicating with its partner on the other node.
TensorFlow Benchmarks
← OLD, NEW →
The figure above shows effect of increasing batch size on training rate. Trying to run Resnet 50 v1.5 with batch size 256 using fp32 for calculations results in OOM, which is why that result is omitted.
← OLD, NEW →
The figure above shows effect of increasing number of gpus on training rate with fixed batch size for both real (blue) and synthetic (orange) data.
...
A broader set of results can be seen in the following table, with results accumulated from 5 separate trials:
model | num_gpus | batch_size | use_fp16 | use_synth | avg img_per_sec | std dev img_per_sec |
---|---|---|---|---|---|---|
resnet50_v1.5 | 1 | 32 | False | False | 300.7115095987747 |
0. |
8334252017088293 | |||||
resnet50_v1.5 | 1 | 32 | False | True | 303.2748894498352 |
0. |
518768277442615 | |||||
resnet50_v1.5 | 1 | 32 | True | False | 552.8543259023911 |
3. |
7419280205833876 | |||||
resnet50_v1.5 | 1 | 32 | True | True | 551. |
3152982879091 | 1.9256151492887408 | |||
resnet50_v1.5 | 1 | 64 | False | False |
340.3455581202521 | 0.4352822822221454 | ||||
resnet50_v1.5 | 1 | 64 | False | True | 341.4178425579963 |
1. |
399820974317093 | |||||
resnet50_v1.5 | 1 | 64 | True | False | 679.092793370331 |
1. |
811493519714515 | |||||
resnet50_v1.5 | 1 | 64 | True | True | 679.4220076333565 |
0. |
4846137767752205 | ||||||
resnet50_v1.5 | 1 | 128 | False | False | 365.97794971142156 | 0. |
6660901988191761 | ||||
resnet50_v1.5 | 1 | 128 | False | True |
365.1002182169415 | 0.290659074503093 | |||||
resnet50_v1.5 | 1 | 128 | True | False | 780.7795427908641 | 2. |
6289190299425735 | |||||
resnet50_v1.5 | 1 | 128 | True | True | 782.0787833811031 |
1. |
4035397167521415 | ||||
resnet50_v1.5 | 1 | 256 | True | False |
841.0209098895596 | 0.9327257479352289 | ||||
resnet50_v1.5 | 1 | 256 | True | True | 841.6281134422322 |
0. |
6754027617726148 | |||||
resnet50_v1.5 | 2 | 32 | False | False | 597.7123015353376 |
2. |
5358048324228486 | ||||
resnet50_v1.5 | 2 | 32 | False | True |
602.1851277710804 | 1.4932197457537746 | ||||
resnet50_v1.5 | 2 | 32 | True | False | 1092.0792847503365 |
5. |
317212492608127 | |||||
resnet50_v1.5 | 2 | 32 | True | True | 1101.2976316254073 |
3. |
1791943633783832 | |||||
resnet50_v1.5 | 2 | 64 | False | False | 680.4331810776715 |
1. |
1986421991311502 | |||||
resnet50_v1.5 | 2 | 64 | False | True | 680.168662439279 |
1. |
628589727697411 | |||||
resnet50_v1.5 | 2 | 64 | True | False | 1350.6915781119133 |
4. |
771936223937959 | |||||
resnet50_v1.5 | 2 | 64 | True | True | 1337.9212057737357 |
4. |
3318244554829635 | ||||
resnet50_v1.5 | 2 | 128 | False | False |
722.1292515998915 | 1.7079082542095976 | |||
resnet50_v1.5 | 2 | 128 | False | True |
724.5907650506392 | 1.3016121917621204 | ||||
resnet50_v1.5 | 2 | 128 | True | False | 1560.6063143166916 |
5. |
574390740438405 | |||||
resnet50_v1.5 | 2 | 128 | True | True | 1558.8925541664316 |
3. |
7645037153069234 | |||||
resnet50_v1.5 | 2 | 256 | True | False | 1681.9367020855304 |
2. |
135490739835972 | |||||
resnet50_v1.5 | 2 | 256 | True | True | 1679.1043258972677 |
1. |
9428881159794065 | ||||
resnet50_v1.5 | 4 | 32 | False | False |
1137.8151032757787 | 3.6070602577235196 | |||
resnet50_v1.5 | 4 | 32 | False | True |
1147.7653259816507 | 10.086122200344256 | ||||
resnet50_v1.5 | 4 | 32 | True | False | 2090.181533002181 |
27. |
753203322924346 | |||||
resnet50_v1.5 | 4 | 32 | True | True | 2118.951749390374 |
9. |
671318775529995 | |||||
resnet50_v1.5 | 4 | 64 | False | False | 1323.4940363814712 |
2. |
4288696630804347 | |||||
resnet50_v1.5 | 4 | 64 | False | True | 1336.5457321023537 |
2. |
8028756633168586 | ||||
resnet50_v1.5 | 4 | 64 | True | False |
2627.208652367316 | 30.590239282719534 | |||
resnet50_v1.5 | 4 | 64 | True | True |
2605.2372571940177 | 16.77388699379653 | ||||
resnet50_v1.5 | 4 | 128 | False | False | 1428.9004479146274 |
5. |
19872681737769 | |||||
resnet50_v1.5 | 4 | 128 | False | True | 1431.0649693247092 |
2. |
1140696182827456 | |||||
resnet50_v1.5 | 4 | 128 | True | False | 3038.123973486709 |
32. |
14983993002834 | |||||
resnet50_v1.5 | 4 | 128 | True | True | 3050.6240630834754 |
13. |
087954923831944 | |||||
resnet50_v1.5 | 4 | 256 | True | False | 3269.060930519893 |
18. |
13964231626883 | ||||
resnet50_v1.5 | 4 | 256 | True | True |
3288. |
inception3
1
32
False
False
0027951477073 | 10.833033249948164 |
inception3 | 1 | 32 | False |
False |
229. |
6400062758412 | 1 |
32
True
False
.011150139067665 | ||
inception3 | 1 | 32 |
True
True
397.61870298268843
inception3
1
64
inception3
1
64
False
False |
False
255.79774049318286
True |
231. |
inception3
1
64
True
False
35776988245647 | 0.7390853739290063 |
inception3 | 1 |
64
32 | True |
inception3
1
128
False |
False
265.4893643297776
inception3
1
128
False
True
390.8430427702688 | 1.5232497382000254 |
inception3
1
inception3 | 1 |
128
True
False
523.6175139095815
32 | True | True |
392. |
inception3
1
256
True
False
77650557982815 | 0.7988964068960409 |
inception3 | 1 |
256
True
True
558.6182587782852
inception3
2
32
64 | False | False |
254. |
inception3
2
32
False
True
08237225221546 | 0.7822494442939251 |
inception3
2
32
True
True
803.1606840744608
inception3
2
64
False
False
505.62156764514185
inception3
2
inception3 |
2
32
True
False
776.9118181766349
1 |
64 | False | True |
254. |
inception3
2
64
True
False
54280397180582 | 0.9677860725157185 |
inception3 |
1 | 64 | True |
True
941.0765499887631
inception3
2
128
False |
False
459. |
inception3
2
128
False
True
532.0966824977393
inception3
2
128
True
False
1031.042745566893
inception3
2
128
True
True
1059.298960623237
inception3
2
256
True
False
1097.3179582016833
inception3
2
256
True
True
1112.944630855092
inception3
4
32
False
False
846.1137665298596
inception3
4
32
False
True
895.5092904990373
inception3
4
32
True
False
1271.1921617960843
inception3
4
32
True
True
1562.0603997601588
inception3
4
64
False
False
967.953751912867
inception3
4
64
False
True
994.9078067162436
inception3
4
64
True
False
1513.182913280423
inception3
4
64
True
True
1842.119923117435
inception3
27229023783576 | 1.0561819758930817 | |||||
inception3 | 1 | 64 | True | True | 465.8370224010511 | 1.9406403927259364 |
inception3 | 1 |
128 | False | False |
266. |
inception3
4
128
False
True
6960069340822 | 0.6349003776029812 |
inception3 |
1 | 128 |
False |
1751.5253071397437
inception3
4
True |
True
267. |
inception3
4
256
True
False
3287620209714 | 0.3378865479524414 |
inception3 |
1 |
256
128 | True |
2174.329529373497
An updated table (utilizing full compute resources / without filesystem bottleneck) with results accumulated from 5 separate trials follows:
False | 522.7059605744032 | 1.2095275730192268 | ||||
inception3 | 1 | 128 | True | True | 528.411119339248 | 1.886616179173554 |
inception3 | 1 | 256 | True | False | 555.9733275297333 | 0.793152246051491 |
inception3 | 1 | 256 | True | True | 555.8605933224619 | 1.0050188430980673 |
inception3 | 2 | 32 | False | False | 453.8981275778566 | 1.4942589199911878 |
inception3 | 2 | 32 | False | True | 460.07629649023164 | 2.572961501771087 |
inception3 | 2 | 32 | True | False | 785.3002107622603 | 2.808072532884698 |
inception3 | ||||||
model | num_gpus | batch_size | use_fp16 | use_synth | mean of img_per_sec | std deviation of img_per_sec |
---|---|---|---|---|---|---|
resnet50_v1.5 | 1 | 32 | False | False | 300.7115095987747 | 0.8334252017088293 |
resnet50_v1.5 | 1 | 32 | False | True | 303.2748894498352 | 0.518768277442615 |
resnet50_v1.5 | 1 | 32 | True | False | 552.8543259023911 | 3.7419280205833876 |
resnet50_v1.5 | 1 | 32 | True | True | 551.3152982879091 | 1.9256151492887408 |
resnet50_v1.5 | 1 | 64 | False | False | 340.3455581202521 | 0.4352822822221454 |
resnet50_v1.5 | 1 | 64 | False | True | 341.4178425579963 | 1.399820974317093 |
resnet50_v1.5 | 1 | 64 | True | False | 679.092793370331 | 1.811493519714515 |
resnet50_v1.5 | 1 | 64 | True | True | 679.4220076333565 | 0.4846137767752205 |
resnet50_v1.5 | 1 | 128 | False | False | 365.97794971142156 | 0.6660901988191761 |
resnet50_v1.5 | 1 | 128 | False | True | 365.1002182169415 | 0.290659074503093 |
resnet50_v1.5 | 1 | 128 | True | False | 780.7795427908641 | 2.6289190299425735 |
resnet50_v1.5 | 1 | 128 | True | True | 782.0787833811031 | 1.4035397167521415 |
resnet50_v1.5 | 1 | 256 | True | False | 841.0209098895596 | 0.9327257479352289 |
resnet50_v1.5 | 1 | 256 | True | True | 841.6281134422322 | 0.6754027617726148 |
resnet50_v1.5 | 2 | 32 | False | False | 597.7123015353376 | 2.5358048324228486 |
resnet50_v1.5 | 2 | 32 | False | True | 602.1851277710804 | 1.4932197457537746 |
resnet50_v1.5 | 2 | 32 | True | True | 1101794.29763162540739652159658681 | 3.1791943633783832resnet50612256492187289 |
inception3_v1.5 | 2 | 64 | False | False | 680504.43318107767159115747109531 | 10.19864219913115029842426876013127 |
inception3resnet50_v1.5 | 2 | 64 | False | True | 680505.1686624392798260905072816 | 1.628589727697411resnet504738932419714106 |
inception3_v1.5 | 2 | 64 | True | False | 1350920.69157811191334145181495171 | 43.7719362239379591651197427361715 |
inception3resnet50_v1.5 | 2 | 64 | True | True | 1337929.92120577373570658731171085 | 45.3318244554829635145909922759968 |
inception3resnet50_v1.5 | 2 | 128 | False | False | 722531.12925159989155809808901386 | 14.7079082542095976152618166592716 |
inception3resnet50_v1.5 | 2 | 128 | False | True | 724531.59076505063923011081204222 | 13.3016121917621204821408673513115 |
inception3resnet50_v1.5 | 2 | 128 | True | False | 15601043.60631431669163751588125951 | 56.574390740438405360344252646946 |
inception3resnet50_v1.5 | 2 | 128 | True | True | 15581052.89255416643161427785292055 | 30.76450371530692348095010705701331 |
inception3resnet50_v1.5 | 2 | 256 | True | False | 16811108.93670208553049748739263916 | 29.135490739835972386603241427991 |
inception3resnet50_v1.5 | 2 | 256 | True | True | 16791113.104325897267716626158571 | 16.94288811597940654319767570544535 |
inception3resnet50_v1.5 | 4 | 32 | False | False | 1137872.81510327577873219766840209 | 34.6070602577235196856647166737354 |
inception3resnet50_v1.5 | 4 | 32 | False | True | 1147888.76532598165073055054654682 | 102.0861222003442565289892419845943 |
inception3resnet50_v1.5 | 4 | 32 | True | False | 20901495.1815330021810846768394943 | 2710.753203322924346965843253024328 |
inception3resnet50_v1.5 | 4 | 32 | True | True | 21181539.9517493903745491706395067 | 9.671318775529995resnet50667722873324415 |
inception3_v1.5 | 4 | 64 | False | False | 1323992.49403638147128041857703738 | 2.4288696630804347resnet504128076229880566 |
inception3_v1.5 | 4 | 64 | False | True | 1336999.54573210235372568604412305 | 2.8028756633168586resnet509532997352535655 |
inception3_v1.5 | 4 | 64 | True | False | 26271790.2086523673167335622916307 | 3012.590239282719534568524155858082 |
inception3resnet50_v1.5 | 4 | 64 | True | True | 26051813.2372571940177338802302623 | 161.773886993796537710531654085244 |
inception3resnet50_v1.5 | 4 | 128 | False | False | 14281044.9004479146274764217747176 | 56.19872681737769219768494743513 |
inception3resnet50_v1.5 | 4 | 128 | False | True | 14311041.06496932470929846289297782 | 216.114069618282745616776883872252 |
inception3resnet50_v1.5 | 4 | 128 | True | False | 30382045.1239734867092687789799281 | 3215.14983993002834112275686406946 |
inception3resnet50_v1.5 | 4 | 128 | True | True | 30502063.6240630834754858653134279 | 138.08795492383194419186760727177 |
inception3resnet50_v1.5 | 4 | 256 | True | False | 32692186.060930519893246302564507 | 1832.13964231626883645185806377874 |
inception3resnet50_v1.5 | 4 | 256 | True | True | 32882194.0027951477073476607046693 | 1020.833033249948164772815631509243 |