# nThread 1 nGpus 1 minBytes 8 maxBytes 8589934592 step: 2(factor) warmup iters: 1 iters: 2 agg iters: 1 validation: 1 graph: 0
#
# Using devices
#  Rank  0 Group  0 Pid 955292 on cw-prod-a100-cu117-3 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank  1 Group  0 Pid 955293 on cw-prod-a100-cu117-3 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank  2 Group  0 Pid 955294 on cw-prod-a100-cu117-3 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank  3 Group  0 Pid 955295 on cw-prod-a100-cu117-3 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank  4 Group  0 Pid 955296 on cw-prod-a100-cu117-3 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank  5 Group  0 Pid 955297 on cw-prod-a100-cu117-3 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank  6 Group  0 Pid 955298 on cw-prod-a100-cu117-3 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank  7 Group  0 Pid 955299 on cw-prod-a100-cu117-3 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank  8 Group  0 Pid 1098248 on cw-prod-a100-cu117-21 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank  9 Group  0 Pid 1098249 on cw-prod-a100-cu117-21 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 10 Group  0 Pid 1098250 on cw-prod-a100-cu117-21 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 11 Group  0 Pid 1098251 on cw-prod-a100-cu117-21 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 12 Group  0 Pid 1098252 on cw-prod-a100-cu117-21 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 13 Group  0 Pid 1098253 on cw-prod-a100-cu117-21 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 14 Group  0 Pid 1098254 on cw-prod-a100-cu117-21 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 15 Group  0 Pid 1098255 on cw-prod-a100-cu117-21 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 16 Group  0 Pid 128709 on cw-prod-a100-cu117-22 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 17 Group  0 Pid 128710 on cw-prod-a100-cu117-22 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 18 Group  0 Pid 128711 on cw-prod-a100-cu117-22 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 19 Group  0 Pid 128712 on cw-prod-a100-cu117-22 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 20 Group  0 Pid 128713 on cw-prod-a100-cu117-22 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 21 Group  0 Pid 128714 on cw-prod-a100-cu117-22 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 22 Group  0 Pid 128715 on cw-prod-a100-cu117-22 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 23 Group  0 Pid 128716 on cw-prod-a100-cu117-22 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 24 Group  0 Pid 1567207 on cw-prod-a100-cu117-23 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 25 Group  0 Pid 1567208 on cw-prod-a100-cu117-23 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 26 Group  0 Pid 1567209 on cw-prod-a100-cu117-23 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 27 Group  0 Pid 1567210 on cw-prod-a100-cu117-23 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 28 Group  0 Pid 1567211 on cw-prod-a100-cu117-23 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 29 Group  0 Pid 1567212 on cw-prod-a100-cu117-23 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 30 Group  0 Pid 1567213 on cw-prod-a100-cu117-23 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 31 Group  0 Pid 1567214 on cw-prod-a100-cu117-23 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 32 Group  0 Pid 1564438 on cw-prod-a100-cu117-24 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 33 Group  0 Pid 1564439 on cw-prod-a100-cu117-24 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 34 Group  0 Pid 1564440 on cw-prod-a100-cu117-24 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 35 Group  0 Pid 1564441 on cw-prod-a100-cu117-24 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 36 Group  0 Pid 1564442 on cw-prod-a100-cu117-24 device  4 [0xc9] NVIDIA A100-SXM4-80GB
#  Rank 37 Group  0 Pid 1564443 on cw-prod-a100-cu117-24 device  5 [0xce] NVIDIA A100-SXM4-80GB
#  Rank 38 Group  0 Pid 1564444 on cw-prod-a100-cu117-24 device  6 [0xe7] NVIDIA A100-SXM4-80GB
#  Rank 39 Group  0 Pid 1564445 on cw-prod-a100-cu117-24 device  7 [0xed] NVIDIA A100-SXM4-80GB
#  Rank 40 Group  0 Pid 1160409 on cw-prod-a100-cu117-25 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 41 Group  0 Pid 1160410 on cw-prod-a100-cu117-25 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 42 Group  0 Pid 1160411 on cw-prod-a100-cu117-25 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 43 Group  0 Pid 1160412 on cw-prod-a100-cu117-25 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 44 Group  0 Pid 1160413 on cw-prod-a100-cu117-25 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 45 Group  0 Pid 1160414 on cw-prod-a100-cu117-25 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 46 Group  0 Pid 1160415 on cw-prod-a100-cu117-25 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 47 Group  0 Pid 1160416 on cw-prod-a100-cu117-25 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 48 Group  0 Pid 1182996 on cw-prod-a100-cu117-27 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 49 Group  0 Pid 1182997 on cw-prod-a100-cu117-27 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 50 Group  0 Pid 1182998 on cw-prod-a100-cu117-27 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 51 Group  0 Pid 1182999 on cw-prod-a100-cu117-27 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 52 Group  0 Pid 1183000 on cw-prod-a100-cu117-27 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 53 Group  0 Pid 1183001 on cw-prod-a100-cu117-27 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 54 Group  0 Pid 1183002 on cw-prod-a100-cu117-27 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 55 Group  0 Pid 1183003 on cw-prod-a100-cu117-27 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 56 Group  0 Pid 1694441 on cw-prod-a100-cu117-36 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 57 Group  0 Pid 1694442 on cw-prod-a100-cu117-36 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 58 Group  0 Pid 1694443 on cw-prod-a100-cu117-36 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 59 Group  0 Pid 1694444 on cw-prod-a100-cu117-36 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 60 Group  0 Pid 1694445 on cw-prod-a100-cu117-36 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 61 Group  0 Pid 1694446 on cw-prod-a100-cu117-36 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 62 Group  0 Pid 1694447 on cw-prod-a100-cu117-36 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 63 Group  0 Pid 1694448 on cw-prod-a100-cu117-36 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 64 Group  0 Pid 1006415 on cw-prod-a100-cu117-38 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 65 Group  0 Pid 1006416 on cw-prod-a100-cu117-38 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 66 Group  0 Pid 1006417 on cw-prod-a100-cu117-38 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 67 Group  0 Pid 1006418 on cw-prod-a100-cu117-38 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 68 Group  0 Pid 1006419 on cw-prod-a100-cu117-38 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 69 Group  0 Pid 1006420 on cw-prod-a100-cu117-38 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 70 Group  0 Pid 1006421 on cw-prod-a100-cu117-38 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 71 Group  0 Pid 1006422 on cw-prod-a100-cu117-38 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 72 Group  0 Pid  82382 on cw-prod-a100-cu117-34 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 73 Group  0 Pid  82383 on cw-prod-a100-cu117-34 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 74 Group  0 Pid  82384 on cw-prod-a100-cu117-34 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 75 Group  0 Pid  82385 on cw-prod-a100-cu117-34 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 76 Group  0 Pid  82386 on cw-prod-a100-cu117-34 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 77 Group  0 Pid  82387 on cw-prod-a100-cu117-34 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 78 Group  0 Pid  82388 on cw-prod-a100-cu117-34 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 79 Group  0 Pid  82389 on cw-prod-a100-cu117-34 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 80 Group  0 Pid 1222320 on cw-prod-a100-cu117-37 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 81 Group  0 Pid 1222321 on cw-prod-a100-cu117-37 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 82 Group  0 Pid 1222322 on cw-prod-a100-cu117-37 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 83 Group  0 Pid 1222323 on cw-prod-a100-cu117-37 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 84 Group  0 Pid 1222324 on cw-prod-a100-cu117-37 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 85 Group  0 Pid 1222325 on cw-prod-a100-cu117-37 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 86 Group  0 Pid 1222326 on cw-prod-a100-cu117-37 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 87 Group  0 Pid 1222327 on cw-prod-a100-cu117-37 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 88 Group  0 Pid 1227608 on cw-prod-a100-cu117-60 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 89 Group  0 Pid 1227609 on cw-prod-a100-cu117-60 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 90 Group  0 Pid 1227610 on cw-prod-a100-cu117-60 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 91 Group  0 Pid 1227611 on cw-prod-a100-cu117-60 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 92 Group  0 Pid 1227612 on cw-prod-a100-cu117-60 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 93 Group  0 Pid 1227613 on cw-prod-a100-cu117-60 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 94 Group  0 Pid 1227614 on cw-prod-a100-cu117-60 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 95 Group  0 Pid 1227615 on cw-prod-a100-cu117-60 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 96 Group  0 Pid 110651 on cw-prod-a100-cu117-28 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 97 Group  0 Pid 110652 on cw-prod-a100-cu117-28 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 98 Group  0 Pid 110653 on cw-prod-a100-cu117-28 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 99 Group  0 Pid 110654 on cw-prod-a100-cu117-28 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 100 Group  0 Pid 110655 on cw-prod-a100-cu117-28 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 101 Group  0 Pid 110656 on cw-prod-a100-cu117-28 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 102 Group  0 Pid 110657 on cw-prod-a100-cu117-28 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 103 Group  0 Pid 110658 on cw-prod-a100-cu117-28 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 104 Group  0 Pid 107684 on cw-prod-a100-cu117-12 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 105 Group  0 Pid 107685 on cw-prod-a100-cu117-12 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 106 Group  0 Pid 107686 on cw-prod-a100-cu117-12 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 107 Group  0 Pid 107687 on cw-prod-a100-cu117-12 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 108 Group  0 Pid 107688 on cw-prod-a100-cu117-12 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 109 Group  0 Pid 107689 on cw-prod-a100-cu117-12 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 110 Group  0 Pid 107690 on cw-prod-a100-cu117-12 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 111 Group  0 Pid 107691 on cw-prod-a100-cu117-12 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 112 Group  0 Pid 1270896 on cw-prod-a100-cu117-35 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 113 Group  0 Pid 1270897 on cw-prod-a100-cu117-35 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 114 Group  0 Pid 1270898 on cw-prod-a100-cu117-35 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 115 Group  0 Pid 1270899 on cw-prod-a100-cu117-35 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 116 Group  0 Pid 1270900 on cw-prod-a100-cu117-35 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 117 Group  0 Pid 1270901 on cw-prod-a100-cu117-35 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 118 Group  0 Pid 1270902 on cw-prod-a100-cu117-35 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 119 Group  0 Pid 1270903 on cw-prod-a100-cu117-35 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 120 Group  0 Pid 1181289 on cw-prod-a100-cu117-44 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 121 Group  0 Pid 1181290 on cw-prod-a100-cu117-44 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 122 Group  0 Pid 1181291 on cw-prod-a100-cu117-44 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 123 Group  0 Pid 1181292 on cw-prod-a100-cu117-44 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 124 Group  0 Pid 1181293 on cw-prod-a100-cu117-44 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 125 Group  0 Pid 1181294 on cw-prod-a100-cu117-44 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 126 Group  0 Pid 1181295 on cw-prod-a100-cu117-44 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 127 Group  0 Pid 1181296 on cw-prod-a100-cu117-44 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 128 Group  0 Pid 1730784 on cw-prod-a100-cu117-40 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 129 Group  0 Pid 1730785 on cw-prod-a100-cu117-40 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 130 Group  0 Pid 1730786 on cw-prod-a100-cu117-40 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 131 Group  0 Pid 1730787 on cw-prod-a100-cu117-40 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 132 Group  0 Pid 1730788 on cw-prod-a100-cu117-40 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 133 Group  0 Pid 1730789 on cw-prod-a100-cu117-40 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 134 Group  0 Pid 1730790 on cw-prod-a100-cu117-40 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 135 Group  0 Pid 1730791 on cw-prod-a100-cu117-40 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 136 Group  0 Pid 131025 on cw-prod-a100-cu117-26 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 137 Group  0 Pid 131026 on cw-prod-a100-cu117-26 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 138 Group  0 Pid 131027 on cw-prod-a100-cu117-26 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 139 Group  0 Pid 131028 on cw-prod-a100-cu117-26 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 140 Group  0 Pid 131029 on cw-prod-a100-cu117-26 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 141 Group  0 Pid 131030 on cw-prod-a100-cu117-26 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 142 Group  0 Pid 131031 on cw-prod-a100-cu117-26 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 143 Group  0 Pid 131032 on cw-prod-a100-cu117-26 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 144 Group  0 Pid 1730120 on cw-prod-a100-cu117-53 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 145 Group  0 Pid 1730121 on cw-prod-a100-cu117-53 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 146 Group  0 Pid 1730122 on cw-prod-a100-cu117-53 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 147 Group  0 Pid 1730123 on cw-prod-a100-cu117-53 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 148 Group  0 Pid 1730124 on cw-prod-a100-cu117-53 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 149 Group  0 Pid 1730125 on cw-prod-a100-cu117-53 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 150 Group  0 Pid 1730126 on cw-prod-a100-cu117-53 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 151 Group  0 Pid 1730127 on cw-prod-a100-cu117-53 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 152 Group  0 Pid 1400767 on cw-prod-a100-cu117-54 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 153 Group  0 Pid 1400768 on cw-prod-a100-cu117-54 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 154 Group  0 Pid 1400769 on cw-prod-a100-cu117-54 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 155 Group  0 Pid 1400770 on cw-prod-a100-cu117-54 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 156 Group  0 Pid 1400771 on cw-prod-a100-cu117-54 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 157 Group  0 Pid 1400772 on cw-prod-a100-cu117-54 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 158 Group  0 Pid 1400773 on cw-prod-a100-cu117-54 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 159 Group  0 Pid 1400774 on cw-prod-a100-cu117-54 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 160 Group  0 Pid 319497 on cw-prod-a100-cu117-46 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 161 Group  0 Pid 319498 on cw-prod-a100-cu117-46 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 162 Group  0 Pid 319499 on cw-prod-a100-cu117-46 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 163 Group  0 Pid 319500 on cw-prod-a100-cu117-46 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 164 Group  0 Pid 319501 on cw-prod-a100-cu117-46 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 165 Group  0 Pid 319502 on cw-prod-a100-cu117-46 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 166 Group  0 Pid 319503 on cw-prod-a100-cu117-46 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 167 Group  0 Pid 319504 on cw-prod-a100-cu117-46 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 168 Group  0 Pid 1863570 on cw-prod-a100-cu117-57 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 169 Group  0 Pid 1863571 on cw-prod-a100-cu117-57 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 170 Group  0 Pid 1863572 on cw-prod-a100-cu117-57 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 171 Group  0 Pid 1863573 on cw-prod-a100-cu117-57 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 172 Group  0 Pid 1863574 on cw-prod-a100-cu117-57 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 173 Group  0 Pid 1863575 on cw-prod-a100-cu117-57 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 174 Group  0 Pid 1863576 on cw-prod-a100-cu117-57 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 175 Group  0 Pid 1863577 on cw-prod-a100-cu117-57 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 176 Group  0 Pid 1095270 on cw-prod-a100-cu117-59 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 177 Group  0 Pid 1095271 on cw-prod-a100-cu117-59 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 178 Group  0 Pid 1095272 on cw-prod-a100-cu117-59 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 179 Group  0 Pid 1095273 on cw-prod-a100-cu117-59 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 180 Group  0 Pid 1095274 on cw-prod-a100-cu117-59 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 181 Group  0 Pid 1095275 on cw-prod-a100-cu117-59 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 182 Group  0 Pid 1095276 on cw-prod-a100-cu117-59 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 183 Group  0 Pid 1095277 on cw-prod-a100-cu117-59 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 184 Group  0 Pid 1281482 on cw-prod-a100-cu117-42 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 185 Group  0 Pid 1281483 on cw-prod-a100-cu117-42 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 186 Group  0 Pid 1281484 on cw-prod-a100-cu117-42 device  2 [0x4d] NVIDIA A100-SXM4-80GB
#  Rank 187 Group  0 Pid 1281485 on cw-prod-a100-cu117-42 device  3 [0x52] NVIDIA A100-SXM4-80GB
#  Rank 188 Group  0 Pid 1281486 on cw-prod-a100-cu117-42 device  4 [0x8d] NVIDIA A100-SXM4-80GB
#  Rank 189 Group  0 Pid 1281487 on cw-prod-a100-cu117-42 device  5 [0x92] NVIDIA A100-SXM4-80GB
#  Rank 190 Group  0 Pid 1281488 on cw-prod-a100-cu117-42 device  6 [0xc9] NVIDIA A100-SXM4-80GB
#  Rank 191 Group  0 Pid 1281489 on cw-prod-a100-cu117-42 device  7 [0xcf] NVIDIA A100-SXM4-80GB
#  Rank 192 Group  0 Pid 135997 on cw-prod-a100-cu117-47 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 193 Group  0 Pid 135998 on cw-prod-a100-cu117-47 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 194 Group  0 Pid 135999 on cw-prod-a100-cu117-47 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 195 Group  0 Pid 136000 on cw-prod-a100-cu117-47 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 196 Group  0 Pid 136001 on cw-prod-a100-cu117-47 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 197 Group  0 Pid 136002 on cw-prod-a100-cu117-47 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 198 Group  0 Pid 136003 on cw-prod-a100-cu117-47 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 199 Group  0 Pid 136004 on cw-prod-a100-cu117-47 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 200 Group  0 Pid 1500591 on cw-prod-a100-cu117-48 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 201 Group  0 Pid 1500592 on cw-prod-a100-cu117-48 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 202 Group  0 Pid 1500593 on cw-prod-a100-cu117-48 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 203 Group  0 Pid 1500594 on cw-prod-a100-cu117-48 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 204 Group  0 Pid 1500595 on cw-prod-a100-cu117-48 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 205 Group  0 Pid 1500596 on cw-prod-a100-cu117-48 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 206 Group  0 Pid 1500597 on cw-prod-a100-cu117-48 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 207 Group  0 Pid 1500598 on cw-prod-a100-cu117-48 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 208 Group  0 Pid 1245567 on cw-prod-a100-cu117-45 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 209 Group  0 Pid 1245568 on cw-prod-a100-cu117-45 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 210 Group  0 Pid 1245569 on cw-prod-a100-cu117-45 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 211 Group  0 Pid 1245570 on cw-prod-a100-cu117-45 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 212 Group  0 Pid 1245571 on cw-prod-a100-cu117-45 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 213 Group  0 Pid 1245572 on cw-prod-a100-cu117-45 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 214 Group  0 Pid 1245573 on cw-prod-a100-cu117-45 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 215 Group  0 Pid 1245574 on cw-prod-a100-cu117-45 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 216 Group  0 Pid 1244907 on cw-prod-a100-cu117-6 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 217 Group  0 Pid 1244908 on cw-prod-a100-cu117-6 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 218 Group  0 Pid 1244909 on cw-prod-a100-cu117-6 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 219 Group  0 Pid 1244910 on cw-prod-a100-cu117-6 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 220 Group  0 Pid 1244911 on cw-prod-a100-cu117-6 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 221 Group  0 Pid 1244912 on cw-prod-a100-cu117-6 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 222 Group  0 Pid 1244913 on cw-prod-a100-cu117-6 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 223 Group  0 Pid 1244914 on cw-prod-a100-cu117-6 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 224 Group  0 Pid 123641 on cw-prod-a100-cu117-41 device  0 [0x11] NVIDIA A100-SXM4-80GB
#  Rank 225 Group  0 Pid 123642 on cw-prod-a100-cu117-41 device  1 [0x17] NVIDIA A100-SXM4-80GB
#  Rank 226 Group  0 Pid 123643 on cw-prod-a100-cu117-41 device  2 [0x4e] NVIDIA A100-SXM4-80GB
#  Rank 227 Group  0 Pid 123644 on cw-prod-a100-cu117-41 device  3 [0x53] NVIDIA A100-SXM4-80GB
#  Rank 228 Group  0 Pid 123645 on cw-prod-a100-cu117-41 device  4 [0x8f] NVIDIA A100-SXM4-80GB
#  Rank 229 Group  0 Pid 123646 on cw-prod-a100-cu117-41 device  5 [0x94] NVIDIA A100-SXM4-80GB
#  Rank 230 Group  0 Pid 123647 on cw-prod-a100-cu117-41 device  6 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 231 Group  0 Pid 123648 on cw-prod-a100-cu117-41 device  7 [0xd0] NVIDIA A100-SXM4-80GB
#  Rank 232 Group  0 Pid 101645 on cw-prod-a100-cu117-63 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 233 Group  0 Pid 101646 on cw-prod-a100-cu117-63 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 234 Group  0 Pid 101647 on cw-prod-a100-cu117-63 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 235 Group  0 Pid 101648 on cw-prod-a100-cu117-63 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 236 Group  0 Pid 101649 on cw-prod-a100-cu117-63 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 237 Group  0 Pid 101650 on cw-prod-a100-cu117-63 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 238 Group  0 Pid 101651 on cw-prod-a100-cu117-63 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 239 Group  0 Pid 101652 on cw-prod-a100-cu117-63 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 240 Group  0 Pid 132936 on cw-prod-a100-cu117-50 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 241 Group  0 Pid 132937 on cw-prod-a100-cu117-50 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 242 Group  0 Pid 132938 on cw-prod-a100-cu117-50 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 243 Group  0 Pid 132939 on cw-prod-a100-cu117-50 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 244 Group  0 Pid 132940 on cw-prod-a100-cu117-50 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 245 Group  0 Pid 132941 on cw-prod-a100-cu117-50 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 246 Group  0 Pid 132942 on cw-prod-a100-cu117-50 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 247 Group  0 Pid 132943 on cw-prod-a100-cu117-50 device  7 [0xca] NVIDIA A100-SXM4-80GB
#  Rank 248 Group  0 Pid 110247 on cw-prod-a100-cu117-20 device  0 [0x27] NVIDIA A100-SXM4-80GB
#  Rank 249 Group  0 Pid 110248 on cw-prod-a100-cu117-20 device  1 [0x2a] NVIDIA A100-SXM4-80GB
#  Rank 250 Group  0 Pid 110249 on cw-prod-a100-cu117-20 device  2 [0x51] NVIDIA A100-SXM4-80GB
#  Rank 251 Group  0 Pid 110250 on cw-prod-a100-cu117-20 device  3 [0x57] NVIDIA A100-SXM4-80GB
#  Rank 252 Group  0 Pid 110251 on cw-prod-a100-cu117-20 device  4 [0x9e] NVIDIA A100-SXM4-80GB
#  Rank 253 Group  0 Pid 110252 on cw-prod-a100-cu117-20 device  5 [0xa4] NVIDIA A100-SXM4-80GB
#  Rank 254 Group  0 Pid 110253 on cw-prod-a100-cu117-20 device  6 [0xc7] NVIDIA A100-SXM4-80GB
#  Rank 255 Group  0 Pid 110254 on cw-prod-a100-cu117-20 device  7 [0xca] NVIDIA A100-SXM4-80GB
#
#                                                              out-of-place                       in-place
#       size         count      type   redop    root     time   algbw   busbw #wrong     time   algbw   busbw #wrong
#        (B)    (elements)                               (us)  (GB/s)  (GB/s)            (us)  (GB/s)  (GB/s)
           8             2     float     sum      -1    128.2    0.00    0.00      0    224.3    0.00    0.00      0
          16             4     float     sum      -1   2719.5    0.00    0.00      0    97.61    0.00    0.00      0
          32             8     float     sum      -1    102.2    0.00    0.00      0    98.11    0.00    0.00      0
          64            16     float     sum      -1    101.5    0.00    0.00      0    91.27    0.00    0.00      0
         128            32     float     sum      -1    123.5    0.00    0.00      0    92.94    0.00    0.00      0
         256            64     float     sum      -1    98.54    0.00    0.01      0    89.74    0.00    0.01      0
         512           128     float     sum      -1    91.32    0.01    0.01      0    92.52    0.01    0.01      0
        1024           256     float     sum      -1    100.1    0.01    0.02      0    120.1    0.01    0.02      0
        2048           512     float     sum      -1    101.0    0.02    0.04      0    104.7    0.02    0.04      0
        4096          1024     float     sum      -1    111.8    0.04    0.07      0    125.8    0.03    0.06      0
        8192          2048     float     sum      -1    135.2    0.06    0.12      0    131.0    0.06    0.12      0
       16384          4096     float     sum      -1    145.8    0.11    0.22      0    173.2    0.09    0.19      0
       32768          8192     float     sum      -1    174.7    0.19    0.37      0    146.6    0.22    0.45      0
       65536         16384     float     sum      -1    159.1    0.41    0.82      0   1735.4    0.04    0.08      0
      131072         32768     float     sum      -1    166.3    0.79    1.57      0    189.3    0.69    1.38      0
      262144         65536     float     sum      -1    158.9    1.65    3.29      0    152.7    1.72    3.42      0
      524288        131072     float     sum      -1    166.5    3.15    6.27      0    170.4    3.08    6.13      0
     1048576        262144     float     sum      -1    199.3    5.26   10.48      0    197.6    5.31   10.57      0
     2097152        524288     float     sum      -1    248.4    8.44   16.82      0    245.9    8.53   16.99      0
     4194304       1048576     float     sum      -1    359.9   11.65   23.22      0    368.9   11.37   22.65      0
     8388608       2097152     float     sum      -1    477.4   17.57   35.00      0    449.9   18.64   37.14      0
    16777216       4194304     float     sum      -1    735.4   22.81   45.45      0    661.6   25.36   50.52      0
    33554432       8388608     float     sum      -1   1220.7   27.49   54.76      0   1145.1   29.30   58.38      0
    67108864      16777216     float     sum      -1   2073.3   32.37   64.48      0   2079.6   32.27   64.29      0
   134217728      33554432     float     sum      -1   3668.5   36.59   72.89      0   3582.4   37.47   74.64      0
   268435456      67108864     float     sum      -1   6541.2   41.04   81.75      0   6560.7   40.92   81.51      0
   536870912     134217728     float     sum      -1    11932   45.00   89.64      0    11954   44.91   89.47      0
  1073741824     268435456     float     sum      -1    22465   47.80   95.22      0    24230   44.31   88.28      0
  2147483648     536870912     float     sum      -1    44040   48.76   97.14      0    44093   48.70   97.03      0
  4294967296    1073741824     float     sum      -1    88079   48.76   97.14      0    87983   48.82   97.25      0
  8589934592    2147483648     float     sum      -1   176310   48.72   97.06      0   175954   48.82   97.26      0
# Out of bounds values : 0 OK
# Avg bus bandwidth    : 28.8996