nccl_log.txt
Never
# nThread 1 nGpus 1 minBytes 8 maxBytes 8589934592 step: 2(factor) warmup iters: 1 iters: 2 agg iters: 1 validation: 1 graph: 0
#
# Using devices
# Rank 0 Group 0 Pid 955292 on cw-prod-a100-cu117-3 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 1 Group 0 Pid 955293 on cw-prod-a100-cu117-3 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 2 Group 0 Pid 955294 on cw-prod-a100-cu117-3 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 3 Group 0 Pid 955295 on cw-prod-a100-cu117-3 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 4 Group 0 Pid 955296 on cw-prod-a100-cu117-3 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 5 Group 0 Pid 955297 on cw-prod-a100-cu117-3 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 6 Group 0 Pid 955298 on cw-prod-a100-cu117-3 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 7 Group 0 Pid 955299 on cw-prod-a100-cu117-3 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 8 Group 0 Pid 1098248 on cw-prod-a100-cu117-21 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 9 Group 0 Pid 1098249 on cw-prod-a100-cu117-21 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 10 Group 0 Pid 1098250 on cw-prod-a100-cu117-21 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 11 Group 0 Pid 1098251 on cw-prod-a100-cu117-21 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 12 Group 0 Pid 1098252 on cw-prod-a100-cu117-21 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 13 Group 0 Pid 1098253 on cw-prod-a100-cu117-21 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 14 Group 0 Pid 1098254 on cw-prod-a100-cu117-21 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 15 Group 0 Pid 1098255 on cw-prod-a100-cu117-21 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 16 Group 0 Pid 128709 on cw-prod-a100-cu117-22 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 17 Group 0 Pid 128710 on cw-prod-a100-cu117-22 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 18 Group 0 Pid 128711 on cw-prod-a100-cu117-22 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 19 Group 0 Pid 128712 on cw-prod-a100-cu117-22 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 20 Group 0 Pid 128713 on cw-prod-a100-cu117-22 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 21 Group 0 Pid 128714 on cw-prod-a100-cu117-22 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 22 Group 0 Pid 128715 on cw-prod-a100-cu117-22 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 23 Group 0 Pid 128716 on cw-prod-a100-cu117-22 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 24 Group 0 Pid 1567207 on cw-prod-a100-cu117-23 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 25 Group 0 Pid 1567208 on cw-prod-a100-cu117-23 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 26 Group 0 Pid 1567209 on cw-prod-a100-cu117-23 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 27 Group 0 Pid 1567210 on cw-prod-a100-cu117-23 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 28 Group 0 Pid 1567211 on cw-prod-a100-cu117-23 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 29 Group 0 Pid 1567212 on cw-prod-a100-cu117-23 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 30 Group 0 Pid 1567213 on cw-prod-a100-cu117-23 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 31 Group 0 Pid 1567214 on cw-prod-a100-cu117-23 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 32 Group 0 Pid 1564438 on cw-prod-a100-cu117-24 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 33 Group 0 Pid 1564439 on cw-prod-a100-cu117-24 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 34 Group 0 Pid 1564440 on cw-prod-a100-cu117-24 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 35 Group 0 Pid 1564441 on cw-prod-a100-cu117-24 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 36 Group 0 Pid 1564442 on cw-prod-a100-cu117-24 device 4 [0xc9] NVIDIA A100-SXM4-80GB
# Rank 37 Group 0 Pid 1564443 on cw-prod-a100-cu117-24 device 5 [0xce] NVIDIA A100-SXM4-80GB
# Rank 38 Group 0 Pid 1564444 on cw-prod-a100-cu117-24 device 6 [0xe7] NVIDIA A100-SXM4-80GB
# Rank 39 Group 0 Pid 1564445 on cw-prod-a100-cu117-24 device 7 [0xed] NVIDIA A100-SXM4-80GB
# Rank 40 Group 0 Pid 1160409 on cw-prod-a100-cu117-25 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 41 Group 0 Pid 1160410 on cw-prod-a100-cu117-25 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 42 Group 0 Pid 1160411 on cw-prod-a100-cu117-25 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 43 Group 0 Pid 1160412 on cw-prod-a100-cu117-25 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 44 Group 0 Pid 1160413 on cw-prod-a100-cu117-25 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 45 Group 0 Pid 1160414 on cw-prod-a100-cu117-25 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 46 Group 0 Pid 1160415 on cw-prod-a100-cu117-25 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 47 Group 0 Pid 1160416 on cw-prod-a100-cu117-25 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 48 Group 0 Pid 1182996 on cw-prod-a100-cu117-27 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 49 Group 0 Pid 1182997 on cw-prod-a100-cu117-27 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 50 Group 0 Pid 1182998 on cw-prod-a100-cu117-27 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 51 Group 0 Pid 1182999 on cw-prod-a100-cu117-27 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 52 Group 0 Pid 1183000 on cw-prod-a100-cu117-27 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 53 Group 0 Pid 1183001 on cw-prod-a100-cu117-27 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 54 Group 0 Pid 1183002 on cw-prod-a100-cu117-27 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 55 Group 0 Pid 1183003 on cw-prod-a100-cu117-27 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 56 Group 0 Pid 1694441 on cw-prod-a100-cu117-36 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 57 Group 0 Pid 1694442 on cw-prod-a100-cu117-36 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 58 Group 0 Pid 1694443 on cw-prod-a100-cu117-36 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 59 Group 0 Pid 1694444 on cw-prod-a100-cu117-36 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 60 Group 0 Pid 1694445 on cw-prod-a100-cu117-36 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 61 Group 0 Pid 1694446 on cw-prod-a100-cu117-36 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 62 Group 0 Pid 1694447 on cw-prod-a100-cu117-36 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 63 Group 0 Pid 1694448 on cw-prod-a100-cu117-36 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 64 Group 0 Pid 1006415 on cw-prod-a100-cu117-38 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 65 Group 0 Pid 1006416 on cw-prod-a100-cu117-38 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 66 Group 0 Pid 1006417 on cw-prod-a100-cu117-38 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 67 Group 0 Pid 1006418 on cw-prod-a100-cu117-38 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 68 Group 0 Pid 1006419 on cw-prod-a100-cu117-38 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 69 Group 0 Pid 1006420 on cw-prod-a100-cu117-38 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 70 Group 0 Pid 1006421 on cw-prod-a100-cu117-38 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 71 Group 0 Pid 1006422 on cw-prod-a100-cu117-38 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 72 Group 0 Pid 82382 on cw-prod-a100-cu117-34 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 73 Group 0 Pid 82383 on cw-prod-a100-cu117-34 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 74 Group 0 Pid 82384 on cw-prod-a100-cu117-34 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 75 Group 0 Pid 82385 on cw-prod-a100-cu117-34 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 76 Group 0 Pid 82386 on cw-prod-a100-cu117-34 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 77 Group 0 Pid 82387 on cw-prod-a100-cu117-34 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 78 Group 0 Pid 82388 on cw-prod-a100-cu117-34 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 79 Group 0 Pid 82389 on cw-prod-a100-cu117-34 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 80 Group 0 Pid 1222320 on cw-prod-a100-cu117-37 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 81 Group 0 Pid 1222321 on cw-prod-a100-cu117-37 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 82 Group 0 Pid 1222322 on cw-prod-a100-cu117-37 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 83 Group 0 Pid 1222323 on cw-prod-a100-cu117-37 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 84 Group 0 Pid 1222324 on cw-prod-a100-cu117-37 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 85 Group 0 Pid 1222325 on cw-prod-a100-cu117-37 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 86 Group 0 Pid 1222326 on cw-prod-a100-cu117-37 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 87 Group 0 Pid 1222327 on cw-prod-a100-cu117-37 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 88 Group 0 Pid 1227608 on cw-prod-a100-cu117-60 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 89 Group 0 Pid 1227609 on cw-prod-a100-cu117-60 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 90 Group 0 Pid 1227610 on cw-prod-a100-cu117-60 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 91 Group 0 Pid 1227611 on cw-prod-a100-cu117-60 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 92 Group 0 Pid 1227612 on cw-prod-a100-cu117-60 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 93 Group 0 Pid 1227613 on cw-prod-a100-cu117-60 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 94 Group 0 Pid 1227614 on cw-prod-a100-cu117-60 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 95 Group 0 Pid 1227615 on cw-prod-a100-cu117-60 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 96 Group 0 Pid 110651 on cw-prod-a100-cu117-28 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 97 Group 0 Pid 110652 on cw-prod-a100-cu117-28 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 98 Group 0 Pid 110653 on cw-prod-a100-cu117-28 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 99 Group 0 Pid 110654 on cw-prod-a100-cu117-28 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 100 Group 0 Pid 110655 on cw-prod-a100-cu117-28 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 101 Group 0 Pid 110656 on cw-prod-a100-cu117-28 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 102 Group 0 Pid 110657 on cw-prod-a100-cu117-28 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 103 Group 0 Pid 110658 on cw-prod-a100-cu117-28 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 104 Group 0 Pid 107684 on cw-prod-a100-cu117-12 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 105 Group 0 Pid 107685 on cw-prod-a100-cu117-12 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 106 Group 0 Pid 107686 on cw-prod-a100-cu117-12 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 107 Group 0 Pid 107687 on cw-prod-a100-cu117-12 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 108 Group 0 Pid 107688 on cw-prod-a100-cu117-12 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 109 Group 0 Pid 107689 on cw-prod-a100-cu117-12 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 110 Group 0 Pid 107690 on cw-prod-a100-cu117-12 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 111 Group 0 Pid 107691 on cw-prod-a100-cu117-12 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 112 Group 0 Pid 1270896 on cw-prod-a100-cu117-35 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 113 Group 0 Pid 1270897 on cw-prod-a100-cu117-35 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 114 Group 0 Pid 1270898 on cw-prod-a100-cu117-35 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 115 Group 0 Pid 1270899 on cw-prod-a100-cu117-35 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 116 Group 0 Pid 1270900 on cw-prod-a100-cu117-35 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 117 Group 0 Pid 1270901 on cw-prod-a100-cu117-35 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 118 Group 0 Pid 1270902 on cw-prod-a100-cu117-35 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 119 Group 0 Pid 1270903 on cw-prod-a100-cu117-35 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 120 Group 0 Pid 1181289 on cw-prod-a100-cu117-44 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 121 Group 0 Pid 1181290 on cw-prod-a100-cu117-44 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 122 Group 0 Pid 1181291 on cw-prod-a100-cu117-44 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 123 Group 0 Pid 1181292 on cw-prod-a100-cu117-44 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 124 Group 0 Pid 1181293 on cw-prod-a100-cu117-44 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 125 Group 0 Pid 1181294 on cw-prod-a100-cu117-44 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 126 Group 0 Pid 1181295 on cw-prod-a100-cu117-44 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 127 Group 0 Pid 1181296 on cw-prod-a100-cu117-44 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 128 Group 0 Pid 1730784 on cw-prod-a100-cu117-40 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 129 Group 0 Pid 1730785 on cw-prod-a100-cu117-40 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 130 Group 0 Pid 1730786 on cw-prod-a100-cu117-40 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 131 Group 0 Pid 1730787 on cw-prod-a100-cu117-40 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 132 Group 0 Pid 1730788 on cw-prod-a100-cu117-40 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 133 Group 0 Pid 1730789 on cw-prod-a100-cu117-40 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 134 Group 0 Pid 1730790 on cw-prod-a100-cu117-40 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 135 Group 0 Pid 1730791 on cw-prod-a100-cu117-40 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 136 Group 0 Pid 131025 on cw-prod-a100-cu117-26 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 137 Group 0 Pid 131026 on cw-prod-a100-cu117-26 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 138 Group 0 Pid 131027 on cw-prod-a100-cu117-26 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 139 Group 0 Pid 131028 on cw-prod-a100-cu117-26 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 140 Group 0 Pid 131029 on cw-prod-a100-cu117-26 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 141 Group 0 Pid 131030 on cw-prod-a100-cu117-26 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 142 Group 0 Pid 131031 on cw-prod-a100-cu117-26 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 143 Group 0 Pid 131032 on cw-prod-a100-cu117-26 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 144 Group 0 Pid 1730120 on cw-prod-a100-cu117-53 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 145 Group 0 Pid 1730121 on cw-prod-a100-cu117-53 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 146 Group 0 Pid 1730122 on cw-prod-a100-cu117-53 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 147 Group 0 Pid 1730123 on cw-prod-a100-cu117-53 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 148 Group 0 Pid 1730124 on cw-prod-a100-cu117-53 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 149 Group 0 Pid 1730125 on cw-prod-a100-cu117-53 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 150 Group 0 Pid 1730126 on cw-prod-a100-cu117-53 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 151 Group 0 Pid 1730127 on cw-prod-a100-cu117-53 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 152 Group 0 Pid 1400767 on cw-prod-a100-cu117-54 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 153 Group 0 Pid 1400768 on cw-prod-a100-cu117-54 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 154 Group 0 Pid 1400769 on cw-prod-a100-cu117-54 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 155 Group 0 Pid 1400770 on cw-prod-a100-cu117-54 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 156 Group 0 Pid 1400771 on cw-prod-a100-cu117-54 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 157 Group 0 Pid 1400772 on cw-prod-a100-cu117-54 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 158 Group 0 Pid 1400773 on cw-prod-a100-cu117-54 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 159 Group 0 Pid 1400774 on cw-prod-a100-cu117-54 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 160 Group 0 Pid 319497 on cw-prod-a100-cu117-46 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 161 Group 0 Pid 319498 on cw-prod-a100-cu117-46 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 162 Group 0 Pid 319499 on cw-prod-a100-cu117-46 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 163 Group 0 Pid 319500 on cw-prod-a100-cu117-46 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 164 Group 0 Pid 319501 on cw-prod-a100-cu117-46 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 165 Group 0 Pid 319502 on cw-prod-a100-cu117-46 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 166 Group 0 Pid 319503 on cw-prod-a100-cu117-46 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 167 Group 0 Pid 319504 on cw-prod-a100-cu117-46 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 168 Group 0 Pid 1863570 on cw-prod-a100-cu117-57 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 169 Group 0 Pid 1863571 on cw-prod-a100-cu117-57 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 170 Group 0 Pid 1863572 on cw-prod-a100-cu117-57 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 171 Group 0 Pid 1863573 on cw-prod-a100-cu117-57 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 172 Group 0 Pid 1863574 on cw-prod-a100-cu117-57 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 173 Group 0 Pid 1863575 on cw-prod-a100-cu117-57 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 174 Group 0 Pid 1863576 on cw-prod-a100-cu117-57 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 175 Group 0 Pid 1863577 on cw-prod-a100-cu117-57 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 176 Group 0 Pid 1095270 on cw-prod-a100-cu117-59 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 177 Group 0 Pid 1095271 on cw-prod-a100-cu117-59 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 178 Group 0 Pid 1095272 on cw-prod-a100-cu117-59 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 179 Group 0 Pid 1095273 on cw-prod-a100-cu117-59 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 180 Group 0 Pid 1095274 on cw-prod-a100-cu117-59 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 181 Group 0 Pid 1095275 on cw-prod-a100-cu117-59 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 182 Group 0 Pid 1095276 on cw-prod-a100-cu117-59 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 183 Group 0 Pid 1095277 on cw-prod-a100-cu117-59 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 184 Group 0 Pid 1281482 on cw-prod-a100-cu117-42 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 185 Group 0 Pid 1281483 on cw-prod-a100-cu117-42 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 186 Group 0 Pid 1281484 on cw-prod-a100-cu117-42 device 2 [0x4d] NVIDIA A100-SXM4-80GB
# Rank 187 Group 0 Pid 1281485 on cw-prod-a100-cu117-42 device 3 [0x52] NVIDIA A100-SXM4-80GB
# Rank 188 Group 0 Pid 1281486 on cw-prod-a100-cu117-42 device 4 [0x8d] NVIDIA A100-SXM4-80GB
# Rank 189 Group 0 Pid 1281487 on cw-prod-a100-cu117-42 device 5 [0x92] NVIDIA A100-SXM4-80GB
# Rank 190 Group 0 Pid 1281488 on cw-prod-a100-cu117-42 device 6 [0xc9] NVIDIA A100-SXM4-80GB
# Rank 191 Group 0 Pid 1281489 on cw-prod-a100-cu117-42 device 7 [0xcf] NVIDIA A100-SXM4-80GB
# Rank 192 Group 0 Pid 135997 on cw-prod-a100-cu117-47 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 193 Group 0 Pid 135998 on cw-prod-a100-cu117-47 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 194 Group 0 Pid 135999 on cw-prod-a100-cu117-47 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 195 Group 0 Pid 136000 on cw-prod-a100-cu117-47 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 196 Group 0 Pid 136001 on cw-prod-a100-cu117-47 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 197 Group 0 Pid 136002 on cw-prod-a100-cu117-47 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 198 Group 0 Pid 136003 on cw-prod-a100-cu117-47 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 199 Group 0 Pid 136004 on cw-prod-a100-cu117-47 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 200 Group 0 Pid 1500591 on cw-prod-a100-cu117-48 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 201 Group 0 Pid 1500592 on cw-prod-a100-cu117-48 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 202 Group 0 Pid 1500593 on cw-prod-a100-cu117-48 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 203 Group 0 Pid 1500594 on cw-prod-a100-cu117-48 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 204 Group 0 Pid 1500595 on cw-prod-a100-cu117-48 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 205 Group 0 Pid 1500596 on cw-prod-a100-cu117-48 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 206 Group 0 Pid 1500597 on cw-prod-a100-cu117-48 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 207 Group 0 Pid 1500598 on cw-prod-a100-cu117-48 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 208 Group 0 Pid 1245567 on cw-prod-a100-cu117-45 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 209 Group 0 Pid 1245568 on cw-prod-a100-cu117-45 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 210 Group 0 Pid 1245569 on cw-prod-a100-cu117-45 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 211 Group 0 Pid 1245570 on cw-prod-a100-cu117-45 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 212 Group 0 Pid 1245571 on cw-prod-a100-cu117-45 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 213 Group 0 Pid 1245572 on cw-prod-a100-cu117-45 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 214 Group 0 Pid 1245573 on cw-prod-a100-cu117-45 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 215 Group 0 Pid 1245574 on cw-prod-a100-cu117-45 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 216 Group 0 Pid 1244907 on cw-prod-a100-cu117-6 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 217 Group 0 Pid 1244908 on cw-prod-a100-cu117-6 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 218 Group 0 Pid 1244909 on cw-prod-a100-cu117-6 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 219 Group 0 Pid 1244910 on cw-prod-a100-cu117-6 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 220 Group 0 Pid 1244911 on cw-prod-a100-cu117-6 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 221 Group 0 Pid 1244912 on cw-prod-a100-cu117-6 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 222 Group 0 Pid 1244913 on cw-prod-a100-cu117-6 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 223 Group 0 Pid 1244914 on cw-prod-a100-cu117-6 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 224 Group 0 Pid 123641 on cw-prod-a100-cu117-41 device 0 [0x11] NVIDIA A100-SXM4-80GB
# Rank 225 Group 0 Pid 123642 on cw-prod-a100-cu117-41 device 1 [0x17] NVIDIA A100-SXM4-80GB
# Rank 226 Group 0 Pid 123643 on cw-prod-a100-cu117-41 device 2 [0x4e] NVIDIA A100-SXM4-80GB
# Rank 227 Group 0 Pid 123644 on cw-prod-a100-cu117-41 device 3 [0x53] NVIDIA A100-SXM4-80GB
# Rank 228 Group 0 Pid 123645 on cw-prod-a100-cu117-41 device 4 [0x8f] NVIDIA A100-SXM4-80GB
# Rank 229 Group 0 Pid 123646 on cw-prod-a100-cu117-41 device 5 [0x94] NVIDIA A100-SXM4-80GB
# Rank 230 Group 0 Pid 123647 on cw-prod-a100-cu117-41 device 6 [0xca] NVIDIA A100-SXM4-80GB
# Rank 231 Group 0 Pid 123648 on cw-prod-a100-cu117-41 device 7 [0xd0] NVIDIA A100-SXM4-80GB
# Rank 232 Group 0 Pid 101645 on cw-prod-a100-cu117-63 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 233 Group 0 Pid 101646 on cw-prod-a100-cu117-63 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 234 Group 0 Pid 101647 on cw-prod-a100-cu117-63 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 235 Group 0 Pid 101648 on cw-prod-a100-cu117-63 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 236 Group 0 Pid 101649 on cw-prod-a100-cu117-63 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 237 Group 0 Pid 101650 on cw-prod-a100-cu117-63 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 238 Group 0 Pid 101651 on cw-prod-a100-cu117-63 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 239 Group 0 Pid 101652 on cw-prod-a100-cu117-63 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 240 Group 0 Pid 132936 on cw-prod-a100-cu117-50 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 241 Group 0 Pid 132937 on cw-prod-a100-cu117-50 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 242 Group 0 Pid 132938 on cw-prod-a100-cu117-50 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 243 Group 0 Pid 132939 on cw-prod-a100-cu117-50 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 244 Group 0 Pid 132940 on cw-prod-a100-cu117-50 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 245 Group 0 Pid 132941 on cw-prod-a100-cu117-50 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 246 Group 0 Pid 132942 on cw-prod-a100-cu117-50 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 247 Group 0 Pid 132943 on cw-prod-a100-cu117-50 device 7 [0xca] NVIDIA A100-SXM4-80GB
# Rank 248 Group 0 Pid 110247 on cw-prod-a100-cu117-20 device 0 [0x27] NVIDIA A100-SXM4-80GB
# Rank 249 Group 0 Pid 110248 on cw-prod-a100-cu117-20 device 1 [0x2a] NVIDIA A100-SXM4-80GB
# Rank 250 Group 0 Pid 110249 on cw-prod-a100-cu117-20 device 2 [0x51] NVIDIA A100-SXM4-80GB
# Rank 251 Group 0 Pid 110250 on cw-prod-a100-cu117-20 device 3 [0x57] NVIDIA A100-SXM4-80GB
# Rank 252 Group 0 Pid 110251 on cw-prod-a100-cu117-20 device 4 [0x9e] NVIDIA A100-SXM4-80GB
# Rank 253 Group 0 Pid 110252 on cw-prod-a100-cu117-20 device 5 [0xa4] NVIDIA A100-SXM4-80GB
# Rank 254 Group 0 Pid 110253 on cw-prod-a100-cu117-20 device 6 [0xc7] NVIDIA A100-SXM4-80GB
# Rank 255 Group 0 Pid 110254 on cw-prod-a100-cu117-20 device 7 [0xca] NVIDIA A100-SXM4-80GB
#
# out-of-place in-place
# size count type redop root time algbw busbw #wrong time algbw busbw #wrong
# (B) (elements) (us) (GB/s) (GB/s) (us) (GB/s) (GB/s)
8 2 float sum -1 128.2 0.00 0.00 0 224.3 0.00 0.00 0
16 4 float sum -1 2719.5 0.00 0.00 0 97.61 0.00 0.00 0
32 8 float sum -1 102.2 0.00 0.00 0 98.11 0.00 0.00 0
64 16 float sum -1 101.5 0.00 0.00 0 91.27 0.00 0.00 0
128 32 float sum -1 123.5 0.00 0.00 0 92.94 0.00 0.00 0
256 64 float sum -1 98.54 0.00 0.01 0 89.74 0.00 0.01 0
512 128 float sum -1 91.32 0.01 0.01 0 92.52 0.01 0.01 0
1024 256 float sum -1 100.1 0.01 0.02 0 120.1 0.01 0.02 0
2048 512 float sum -1 101.0 0.02 0.04 0 104.7 0.02 0.04 0
4096 1024 float sum -1 111.8 0.04 0.07 0 125.8 0.03 0.06 0
8192 2048 float sum -1 135.2 0.06 0.12 0 131.0 0.06 0.12 0
16384 4096 float sum -1 145.8 0.11 0.22 0 173.2 0.09 0.19 0
32768 8192 float sum -1 174.7 0.19 0.37 0 146.6 0.22 0.45 0
65536 16384 float sum -1 159.1 0.41 0.82 0 1735.4 0.04 0.08 0
131072 32768 float sum -1 166.3 0.79 1.57 0 189.3 0.69 1.38 0
262144 65536 float sum -1 158.9 1.65 3.29 0 152.7 1.72 3.42 0
524288 131072 float sum -1 166.5 3.15 6.27 0 170.4 3.08 6.13 0
1048576 262144 float sum -1 199.3 5.26 10.48 0 197.6 5.31 10.57 0
2097152 524288 float sum -1 248.4 8.44 16.82 0 245.9 8.53 16.99 0
4194304 1048576 float sum -1 359.9 11.65 23.22 0 368.9 11.37 22.65 0
8388608 2097152 float sum -1 477.4 17.57 35.00 0 449.9 18.64 37.14 0
16777216 4194304 float sum -1 735.4 22.81 45.45 0 661.6 25.36 50.52 0
33554432 8388608 float sum -1 1220.7 27.49 54.76 0 1145.1 29.30 58.38 0
67108864 16777216 float sum -1 2073.3 32.37 64.48 0 2079.6 32.27 64.29 0
134217728 33554432 float sum -1 3668.5 36.59 72.89 0 3582.4 37.47 74.64 0
268435456 67108864 float sum -1 6541.2 41.04 81.75 0 6560.7 40.92 81.51 0
536870912 134217728 float sum -1 11932 45.00 89.64 0 11954 44.91 89.47 0
1073741824 268435456 float sum -1 22465 47.80 95.22 0 24230 44.31 88.28 0
2147483648 536870912 float sum -1 44040 48.76 97.14 0 44093 48.70 97.03 0
4294967296 1073741824 float sum -1 88079 48.76 97.14 0 87983 48.82 97.25 0
8589934592 2147483648 float sum -1 176310 48.72 97.06 0 175954 48.82 97.26 0
# Out of bounds values : 0 OK
# Avg bus bandwidth : 28.8996
Raw Text
-
Breaking news - Update 1 - 12/09/2023 10:00:14
6 min ago
-
Breaking news - Update 1 - 12/09/2023 09:53:33
9 min ago
-
fdgszdfg
11 min ago
-
Breaking news - Update 1 - 12/09/2023 09:53:49
12 min ago
-
Breaking news - Update 1 - 12/09/2023 09:45:16
21 min ago
-
Breaking news - Update 1 - 12/09/2023 09:41:05
22 min ago
-
Breaking news - Update 1 - 12/09/2023 09:37:54
28 min ago
-
Breaking news - Update 1 - 12/09/2023 09:33:58
29 min ago
-
dsgff
29 min ago
-
Breaking news - Update 1 - 12/09/2023 09:29:41
36 min ago