nccl_log.txt
Never
# nThread 1 nGpus 1 minBytes 8 maxBytes 8589934592 step: 2(factor) warmup iters: 1 iters: 2 agg iters: 1 validation: 1 graph: 0 # # Using devices # Rank 0 Group 0 Pid 955292 on cw-prod-a100-cu117-3 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 1 Group 0 Pid 955293 on cw-prod-a100-cu117-3 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 2 Group 0 Pid 955294 on cw-prod-a100-cu117-3 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 3 Group 0 Pid 955295 on cw-prod-a100-cu117-3 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 4 Group 0 Pid 955296 on cw-prod-a100-cu117-3 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 5 Group 0 Pid 955297 on cw-prod-a100-cu117-3 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 6 Group 0 Pid 955298 on cw-prod-a100-cu117-3 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 7 Group 0 Pid 955299 on cw-prod-a100-cu117-3 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 8 Group 0 Pid 1098248 on cw-prod-a100-cu117-21 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 9 Group 0 Pid 1098249 on cw-prod-a100-cu117-21 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 10 Group 0 Pid 1098250 on cw-prod-a100-cu117-21 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 11 Group 0 Pid 1098251 on cw-prod-a100-cu117-21 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 12 Group 0 Pid 1098252 on cw-prod-a100-cu117-21 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 13 Group 0 Pid 1098253 on cw-prod-a100-cu117-21 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 14 Group 0 Pid 1098254 on cw-prod-a100-cu117-21 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 15 Group 0 Pid 1098255 on cw-prod-a100-cu117-21 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 16 Group 0 Pid 128709 on cw-prod-a100-cu117-22 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 17 Group 0 Pid 128710 on cw-prod-a100-cu117-22 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 18 Group 0 Pid 128711 on cw-prod-a100-cu117-22 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 19 Group 0 Pid 128712 on cw-prod-a100-cu117-22 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 20 Group 0 Pid 128713 on cw-prod-a100-cu117-22 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 21 Group 0 Pid 128714 on cw-prod-a100-cu117-22 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 22 Group 0 Pid 128715 on cw-prod-a100-cu117-22 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 23 Group 0 Pid 128716 on cw-prod-a100-cu117-22 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 24 Group 0 Pid 1567207 on cw-prod-a100-cu117-23 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 25 Group 0 Pid 1567208 on cw-prod-a100-cu117-23 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 26 Group 0 Pid 1567209 on cw-prod-a100-cu117-23 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 27 Group 0 Pid 1567210 on cw-prod-a100-cu117-23 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 28 Group 0 Pid 1567211 on cw-prod-a100-cu117-23 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 29 Group 0 Pid 1567212 on cw-prod-a100-cu117-23 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 30 Group 0 Pid 1567213 on cw-prod-a100-cu117-23 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 31 Group 0 Pid 1567214 on cw-prod-a100-cu117-23 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 32 Group 0 Pid 1564438 on cw-prod-a100-cu117-24 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 33 Group 0 Pid 1564439 on cw-prod-a100-cu117-24 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 34 Group 0 Pid 1564440 on cw-prod-a100-cu117-24 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 35 Group 0 Pid 1564441 on cw-prod-a100-cu117-24 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 36 Group 0 Pid 1564442 on cw-prod-a100-cu117-24 device 4 [0xc9] NVIDIA A100-SXM4-80GB # Rank 37 Group 0 Pid 1564443 on cw-prod-a100-cu117-24 device 5 [0xce] NVIDIA A100-SXM4-80GB # Rank 38 Group 0 Pid 1564444 on cw-prod-a100-cu117-24 device 6 [0xe7] NVIDIA A100-SXM4-80GB # Rank 39 Group 0 Pid 1564445 on cw-prod-a100-cu117-24 device 7 [0xed] NVIDIA A100-SXM4-80GB # Rank 40 Group 0 Pid 1160409 on cw-prod-a100-cu117-25 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 41 Group 0 Pid 1160410 on cw-prod-a100-cu117-25 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 42 Group 0 Pid 1160411 on cw-prod-a100-cu117-25 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 43 Group 0 Pid 1160412 on cw-prod-a100-cu117-25 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 44 Group 0 Pid 1160413 on cw-prod-a100-cu117-25 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 45 Group 0 Pid 1160414 on cw-prod-a100-cu117-25 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 46 Group 0 Pid 1160415 on cw-prod-a100-cu117-25 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 47 Group 0 Pid 1160416 on cw-prod-a100-cu117-25 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 48 Group 0 Pid 1182996 on cw-prod-a100-cu117-27 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 49 Group 0 Pid 1182997 on cw-prod-a100-cu117-27 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 50 Group 0 Pid 1182998 on cw-prod-a100-cu117-27 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 51 Group 0 Pid 1182999 on cw-prod-a100-cu117-27 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 52 Group 0 Pid 1183000 on cw-prod-a100-cu117-27 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 53 Group 0 Pid 1183001 on cw-prod-a100-cu117-27 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 54 Group 0 Pid 1183002 on cw-prod-a100-cu117-27 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 55 Group 0 Pid 1183003 on cw-prod-a100-cu117-27 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 56 Group 0 Pid 1694441 on cw-prod-a100-cu117-36 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 57 Group 0 Pid 1694442 on cw-prod-a100-cu117-36 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 58 Group 0 Pid 1694443 on cw-prod-a100-cu117-36 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 59 Group 0 Pid 1694444 on cw-prod-a100-cu117-36 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 60 Group 0 Pid 1694445 on cw-prod-a100-cu117-36 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 61 Group 0 Pid 1694446 on cw-prod-a100-cu117-36 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 62 Group 0 Pid 1694447 on cw-prod-a100-cu117-36 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 63 Group 0 Pid 1694448 on cw-prod-a100-cu117-36 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 64 Group 0 Pid 1006415 on cw-prod-a100-cu117-38 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 65 Group 0 Pid 1006416 on cw-prod-a100-cu117-38 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 66 Group 0 Pid 1006417 on cw-prod-a100-cu117-38 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 67 Group 0 Pid 1006418 on cw-prod-a100-cu117-38 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 68 Group 0 Pid 1006419 on cw-prod-a100-cu117-38 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 69 Group 0 Pid 1006420 on cw-prod-a100-cu117-38 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 70 Group 0 Pid 1006421 on cw-prod-a100-cu117-38 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 71 Group 0 Pid 1006422 on cw-prod-a100-cu117-38 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 72 Group 0 Pid 82382 on cw-prod-a100-cu117-34 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 73 Group 0 Pid 82383 on cw-prod-a100-cu117-34 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 74 Group 0 Pid 82384 on cw-prod-a100-cu117-34 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 75 Group 0 Pid 82385 on cw-prod-a100-cu117-34 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 76 Group 0 Pid 82386 on cw-prod-a100-cu117-34 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 77 Group 0 Pid 82387 on cw-prod-a100-cu117-34 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 78 Group 0 Pid 82388 on cw-prod-a100-cu117-34 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 79 Group 0 Pid 82389 on cw-prod-a100-cu117-34 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 80 Group 0 Pid 1222320 on cw-prod-a100-cu117-37 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 81 Group 0 Pid 1222321 on cw-prod-a100-cu117-37 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 82 Group 0 Pid 1222322 on cw-prod-a100-cu117-37 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 83 Group 0 Pid 1222323 on cw-prod-a100-cu117-37 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 84 Group 0 Pid 1222324 on cw-prod-a100-cu117-37 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 85 Group 0 Pid 1222325 on cw-prod-a100-cu117-37 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 86 Group 0 Pid 1222326 on cw-prod-a100-cu117-37 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 87 Group 0 Pid 1222327 on cw-prod-a100-cu117-37 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 88 Group 0 Pid 1227608 on cw-prod-a100-cu117-60 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 89 Group 0 Pid 1227609 on cw-prod-a100-cu117-60 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 90 Group 0 Pid 1227610 on cw-prod-a100-cu117-60 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 91 Group 0 Pid 1227611 on cw-prod-a100-cu117-60 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 92 Group 0 Pid 1227612 on cw-prod-a100-cu117-60 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 93 Group 0 Pid 1227613 on cw-prod-a100-cu117-60 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 94 Group 0 Pid 1227614 on cw-prod-a100-cu117-60 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 95 Group 0 Pid 1227615 on cw-prod-a100-cu117-60 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 96 Group 0 Pid 110651 on cw-prod-a100-cu117-28 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 97 Group 0 Pid 110652 on cw-prod-a100-cu117-28 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 98 Group 0 Pid 110653 on cw-prod-a100-cu117-28 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 99 Group 0 Pid 110654 on cw-prod-a100-cu117-28 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 100 Group 0 Pid 110655 on cw-prod-a100-cu117-28 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 101 Group 0 Pid 110656 on cw-prod-a100-cu117-28 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 102 Group 0 Pid 110657 on cw-prod-a100-cu117-28 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 103 Group 0 Pid 110658 on cw-prod-a100-cu117-28 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 104 Group 0 Pid 107684 on cw-prod-a100-cu117-12 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 105 Group 0 Pid 107685 on cw-prod-a100-cu117-12 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 106 Group 0 Pid 107686 on cw-prod-a100-cu117-12 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 107 Group 0 Pid 107687 on cw-prod-a100-cu117-12 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 108 Group 0 Pid 107688 on cw-prod-a100-cu117-12 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 109 Group 0 Pid 107689 on cw-prod-a100-cu117-12 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 110 Group 0 Pid 107690 on cw-prod-a100-cu117-12 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 111 Group 0 Pid 107691 on cw-prod-a100-cu117-12 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 112 Group 0 Pid 1270896 on cw-prod-a100-cu117-35 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 113 Group 0 Pid 1270897 on cw-prod-a100-cu117-35 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 114 Group 0 Pid 1270898 on cw-prod-a100-cu117-35 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 115 Group 0 Pid 1270899 on cw-prod-a100-cu117-35 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 116 Group 0 Pid 1270900 on cw-prod-a100-cu117-35 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 117 Group 0 Pid 1270901 on cw-prod-a100-cu117-35 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 118 Group 0 Pid 1270902 on cw-prod-a100-cu117-35 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 119 Group 0 Pid 1270903 on cw-prod-a100-cu117-35 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 120 Group 0 Pid 1181289 on cw-prod-a100-cu117-44 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 121 Group 0 Pid 1181290 on cw-prod-a100-cu117-44 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 122 Group 0 Pid 1181291 on cw-prod-a100-cu117-44 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 123 Group 0 Pid 1181292 on cw-prod-a100-cu117-44 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 124 Group 0 Pid 1181293 on cw-prod-a100-cu117-44 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 125 Group 0 Pid 1181294 on cw-prod-a100-cu117-44 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 126 Group 0 Pid 1181295 on cw-prod-a100-cu117-44 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 127 Group 0 Pid 1181296 on cw-prod-a100-cu117-44 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 128 Group 0 Pid 1730784 on cw-prod-a100-cu117-40 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 129 Group 0 Pid 1730785 on cw-prod-a100-cu117-40 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 130 Group 0 Pid 1730786 on cw-prod-a100-cu117-40 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 131 Group 0 Pid 1730787 on cw-prod-a100-cu117-40 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 132 Group 0 Pid 1730788 on cw-prod-a100-cu117-40 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 133 Group 0 Pid 1730789 on cw-prod-a100-cu117-40 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 134 Group 0 Pid 1730790 on cw-prod-a100-cu117-40 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 135 Group 0 Pid 1730791 on cw-prod-a100-cu117-40 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 136 Group 0 Pid 131025 on cw-prod-a100-cu117-26 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 137 Group 0 Pid 131026 on cw-prod-a100-cu117-26 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 138 Group 0 Pid 131027 on cw-prod-a100-cu117-26 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 139 Group 0 Pid 131028 on cw-prod-a100-cu117-26 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 140 Group 0 Pid 131029 on cw-prod-a100-cu117-26 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 141 Group 0 Pid 131030 on cw-prod-a100-cu117-26 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 142 Group 0 Pid 131031 on cw-prod-a100-cu117-26 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 143 Group 0 Pid 131032 on cw-prod-a100-cu117-26 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 144 Group 0 Pid 1730120 on cw-prod-a100-cu117-53 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 145 Group 0 Pid 1730121 on cw-prod-a100-cu117-53 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 146 Group 0 Pid 1730122 on cw-prod-a100-cu117-53 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 147 Group 0 Pid 1730123 on cw-prod-a100-cu117-53 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 148 Group 0 Pid 1730124 on cw-prod-a100-cu117-53 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 149 Group 0 Pid 1730125 on cw-prod-a100-cu117-53 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 150 Group 0 Pid 1730126 on cw-prod-a100-cu117-53 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 151 Group 0 Pid 1730127 on cw-prod-a100-cu117-53 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 152 Group 0 Pid 1400767 on cw-prod-a100-cu117-54 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 153 Group 0 Pid 1400768 on cw-prod-a100-cu117-54 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 154 Group 0 Pid 1400769 on cw-prod-a100-cu117-54 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 155 Group 0 Pid 1400770 on cw-prod-a100-cu117-54 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 156 Group 0 Pid 1400771 on cw-prod-a100-cu117-54 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 157 Group 0 Pid 1400772 on cw-prod-a100-cu117-54 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 158 Group 0 Pid 1400773 on cw-prod-a100-cu117-54 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 159 Group 0 Pid 1400774 on cw-prod-a100-cu117-54 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 160 Group 0 Pid 319497 on cw-prod-a100-cu117-46 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 161 Group 0 Pid 319498 on cw-prod-a100-cu117-46 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 162 Group 0 Pid 319499 on cw-prod-a100-cu117-46 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 163 Group 0 Pid 319500 on cw-prod-a100-cu117-46 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 164 Group 0 Pid 319501 on cw-prod-a100-cu117-46 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 165 Group 0 Pid 319502 on cw-prod-a100-cu117-46 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 166 Group 0 Pid 319503 on cw-prod-a100-cu117-46 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 167 Group 0 Pid 319504 on cw-prod-a100-cu117-46 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 168 Group 0 Pid 1863570 on cw-prod-a100-cu117-57 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 169 Group 0 Pid 1863571 on cw-prod-a100-cu117-57 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 170 Group 0 Pid 1863572 on cw-prod-a100-cu117-57 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 171 Group 0 Pid 1863573 on cw-prod-a100-cu117-57 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 172 Group 0 Pid 1863574 on cw-prod-a100-cu117-57 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 173 Group 0 Pid 1863575 on cw-prod-a100-cu117-57 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 174 Group 0 Pid 1863576 on cw-prod-a100-cu117-57 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 175 Group 0 Pid 1863577 on cw-prod-a100-cu117-57 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 176 Group 0 Pid 1095270 on cw-prod-a100-cu117-59 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 177 Group 0 Pid 1095271 on cw-prod-a100-cu117-59 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 178 Group 0 Pid 1095272 on cw-prod-a100-cu117-59 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 179 Group 0 Pid 1095273 on cw-prod-a100-cu117-59 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 180 Group 0 Pid 1095274 on cw-prod-a100-cu117-59 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 181 Group 0 Pid 1095275 on cw-prod-a100-cu117-59 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 182 Group 0 Pid 1095276 on cw-prod-a100-cu117-59 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 183 Group 0 Pid 1095277 on cw-prod-a100-cu117-59 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 184 Group 0 Pid 1281482 on cw-prod-a100-cu117-42 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 185 Group 0 Pid 1281483 on cw-prod-a100-cu117-42 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 186 Group 0 Pid 1281484 on cw-prod-a100-cu117-42 device 2 [0x4d] NVIDIA A100-SXM4-80GB # Rank 187 Group 0 Pid 1281485 on cw-prod-a100-cu117-42 device 3 [0x52] NVIDIA A100-SXM4-80GB # Rank 188 Group 0 Pid 1281486 on cw-prod-a100-cu117-42 device 4 [0x8d] NVIDIA A100-SXM4-80GB # Rank 189 Group 0 Pid 1281487 on cw-prod-a100-cu117-42 device 5 [0x92] NVIDIA A100-SXM4-80GB # Rank 190 Group 0 Pid 1281488 on cw-prod-a100-cu117-42 device 6 [0xc9] NVIDIA A100-SXM4-80GB # Rank 191 Group 0 Pid 1281489 on cw-prod-a100-cu117-42 device 7 [0xcf] NVIDIA A100-SXM4-80GB # Rank 192 Group 0 Pid 135997 on cw-prod-a100-cu117-47 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 193 Group 0 Pid 135998 on cw-prod-a100-cu117-47 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 194 Group 0 Pid 135999 on cw-prod-a100-cu117-47 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 195 Group 0 Pid 136000 on cw-prod-a100-cu117-47 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 196 Group 0 Pid 136001 on cw-prod-a100-cu117-47 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 197 Group 0 Pid 136002 on cw-prod-a100-cu117-47 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 198 Group 0 Pid 136003 on cw-prod-a100-cu117-47 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 199 Group 0 Pid 136004 on cw-prod-a100-cu117-47 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 200 Group 0 Pid 1500591 on cw-prod-a100-cu117-48 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 201 Group 0 Pid 1500592 on cw-prod-a100-cu117-48 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 202 Group 0 Pid 1500593 on cw-prod-a100-cu117-48 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 203 Group 0 Pid 1500594 on cw-prod-a100-cu117-48 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 204 Group 0 Pid 1500595 on cw-prod-a100-cu117-48 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 205 Group 0 Pid 1500596 on cw-prod-a100-cu117-48 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 206 Group 0 Pid 1500597 on cw-prod-a100-cu117-48 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 207 Group 0 Pid 1500598 on cw-prod-a100-cu117-48 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 208 Group 0 Pid 1245567 on cw-prod-a100-cu117-45 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 209 Group 0 Pid 1245568 on cw-prod-a100-cu117-45 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 210 Group 0 Pid 1245569 on cw-prod-a100-cu117-45 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 211 Group 0 Pid 1245570 on cw-prod-a100-cu117-45 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 212 Group 0 Pid 1245571 on cw-prod-a100-cu117-45 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 213 Group 0 Pid 1245572 on cw-prod-a100-cu117-45 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 214 Group 0 Pid 1245573 on cw-prod-a100-cu117-45 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 215 Group 0 Pid 1245574 on cw-prod-a100-cu117-45 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 216 Group 0 Pid 1244907 on cw-prod-a100-cu117-6 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 217 Group 0 Pid 1244908 on cw-prod-a100-cu117-6 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 218 Group 0 Pid 1244909 on cw-prod-a100-cu117-6 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 219 Group 0 Pid 1244910 on cw-prod-a100-cu117-6 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 220 Group 0 Pid 1244911 on cw-prod-a100-cu117-6 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 221 Group 0 Pid 1244912 on cw-prod-a100-cu117-6 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 222 Group 0 Pid 1244913 on cw-prod-a100-cu117-6 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 223 Group 0 Pid 1244914 on cw-prod-a100-cu117-6 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 224 Group 0 Pid 123641 on cw-prod-a100-cu117-41 device 0 [0x11] NVIDIA A100-SXM4-80GB # Rank 225 Group 0 Pid 123642 on cw-prod-a100-cu117-41 device 1 [0x17] NVIDIA A100-SXM4-80GB # Rank 226 Group 0 Pid 123643 on cw-prod-a100-cu117-41 device 2 [0x4e] NVIDIA A100-SXM4-80GB # Rank 227 Group 0 Pid 123644 on cw-prod-a100-cu117-41 device 3 [0x53] NVIDIA A100-SXM4-80GB # Rank 228 Group 0 Pid 123645 on cw-prod-a100-cu117-41 device 4 [0x8f] NVIDIA A100-SXM4-80GB # Rank 229 Group 0 Pid 123646 on cw-prod-a100-cu117-41 device 5 [0x94] NVIDIA A100-SXM4-80GB # Rank 230 Group 0 Pid 123647 on cw-prod-a100-cu117-41 device 6 [0xca] NVIDIA A100-SXM4-80GB # Rank 231 Group 0 Pid 123648 on cw-prod-a100-cu117-41 device 7 [0xd0] NVIDIA A100-SXM4-80GB # Rank 232 Group 0 Pid 101645 on cw-prod-a100-cu117-63 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 233 Group 0 Pid 101646 on cw-prod-a100-cu117-63 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 234 Group 0 Pid 101647 on cw-prod-a100-cu117-63 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 235 Group 0 Pid 101648 on cw-prod-a100-cu117-63 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 236 Group 0 Pid 101649 on cw-prod-a100-cu117-63 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 237 Group 0 Pid 101650 on cw-prod-a100-cu117-63 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 238 Group 0 Pid 101651 on cw-prod-a100-cu117-63 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 239 Group 0 Pid 101652 on cw-prod-a100-cu117-63 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 240 Group 0 Pid 132936 on cw-prod-a100-cu117-50 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 241 Group 0 Pid 132937 on cw-prod-a100-cu117-50 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 242 Group 0 Pid 132938 on cw-prod-a100-cu117-50 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 243 Group 0 Pid 132939 on cw-prod-a100-cu117-50 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 244 Group 0 Pid 132940 on cw-prod-a100-cu117-50 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 245 Group 0 Pid 132941 on cw-prod-a100-cu117-50 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 246 Group 0 Pid 132942 on cw-prod-a100-cu117-50 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 247 Group 0 Pid 132943 on cw-prod-a100-cu117-50 device 7 [0xca] NVIDIA A100-SXM4-80GB # Rank 248 Group 0 Pid 110247 on cw-prod-a100-cu117-20 device 0 [0x27] NVIDIA A100-SXM4-80GB # Rank 249 Group 0 Pid 110248 on cw-prod-a100-cu117-20 device 1 [0x2a] NVIDIA A100-SXM4-80GB # Rank 250 Group 0 Pid 110249 on cw-prod-a100-cu117-20 device 2 [0x51] NVIDIA A100-SXM4-80GB # Rank 251 Group 0 Pid 110250 on cw-prod-a100-cu117-20 device 3 [0x57] NVIDIA A100-SXM4-80GB # Rank 252 Group 0 Pid 110251 on cw-prod-a100-cu117-20 device 4 [0x9e] NVIDIA A100-SXM4-80GB # Rank 253 Group 0 Pid 110252 on cw-prod-a100-cu117-20 device 5 [0xa4] NVIDIA A100-SXM4-80GB # Rank 254 Group 0 Pid 110253 on cw-prod-a100-cu117-20 device 6 [0xc7] NVIDIA A100-SXM4-80GB # Rank 255 Group 0 Pid 110254 on cw-prod-a100-cu117-20 device 7 [0xca] NVIDIA A100-SXM4-80GB # # out-of-place in-place # size count type redop root time algbw busbw #wrong time algbw busbw #wrong # (B) (elements) (us) (GB/s) (GB/s) (us) (GB/s) (GB/s) 8 2 float sum -1 128.2 0.00 0.00 0 224.3 0.00 0.00 0 16 4 float sum -1 2719.5 0.00 0.00 0 97.61 0.00 0.00 0 32 8 float sum -1 102.2 0.00 0.00 0 98.11 0.00 0.00 0 64 16 float sum -1 101.5 0.00 0.00 0 91.27 0.00 0.00 0 128 32 float sum -1 123.5 0.00 0.00 0 92.94 0.00 0.00 0 256 64 float sum -1 98.54 0.00 0.01 0 89.74 0.00 0.01 0 512 128 float sum -1 91.32 0.01 0.01 0 92.52 0.01 0.01 0 1024 256 float sum -1 100.1 0.01 0.02 0 120.1 0.01 0.02 0 2048 512 float sum -1 101.0 0.02 0.04 0 104.7 0.02 0.04 0 4096 1024 float sum -1 111.8 0.04 0.07 0 125.8 0.03 0.06 0 8192 2048 float sum -1 135.2 0.06 0.12 0 131.0 0.06 0.12 0 16384 4096 float sum -1 145.8 0.11 0.22 0 173.2 0.09 0.19 0 32768 8192 float sum -1 174.7 0.19 0.37 0 146.6 0.22 0.45 0 65536 16384 float sum -1 159.1 0.41 0.82 0 1735.4 0.04 0.08 0 131072 32768 float sum -1 166.3 0.79 1.57 0 189.3 0.69 1.38 0 262144 65536 float sum -1 158.9 1.65 3.29 0 152.7 1.72 3.42 0 524288 131072 float sum -1 166.5 3.15 6.27 0 170.4 3.08 6.13 0 1048576 262144 float sum -1 199.3 5.26 10.48 0 197.6 5.31 10.57 0 2097152 524288 float sum -1 248.4 8.44 16.82 0 245.9 8.53 16.99 0 4194304 1048576 float sum -1 359.9 11.65 23.22 0 368.9 11.37 22.65 0 8388608 2097152 float sum -1 477.4 17.57 35.00 0 449.9 18.64 37.14 0 16777216 4194304 float sum -1 735.4 22.81 45.45 0 661.6 25.36 50.52 0 33554432 8388608 float sum -1 1220.7 27.49 54.76 0 1145.1 29.30 58.38 0 67108864 16777216 float sum -1 2073.3 32.37 64.48 0 2079.6 32.27 64.29 0 134217728 33554432 float sum -1 3668.5 36.59 72.89 0 3582.4 37.47 74.64 0 268435456 67108864 float sum -1 6541.2 41.04 81.75 0 6560.7 40.92 81.51 0 536870912 134217728 float sum -1 11932 45.00 89.64 0 11954 44.91 89.47 0 1073741824 268435456 float sum -1 22465 47.80 95.22 0 24230 44.31 88.28 0 2147483648 536870912 float sum -1 44040 48.76 97.14 0 44093 48.70 97.03 0 4294967296 1073741824 float sum -1 88079 48.76 97.14 0 87983 48.82 97.25 0 8589934592 2147483648 float sum -1 176310 48.72 97.06 0 175954 48.82 97.26 0 # Out of bounds values : 0 OK # Avg bus bandwidth : 28.8996
Raw Text
-
Breaking news - Update 1 - 12/09/2023 10:00:14
6 min ago
-
Breaking news - Update 1 - 12/09/2023 09:53:33
9 min ago
-
fdgszdfg
11 min ago
-
Breaking news - Update 1 - 12/09/2023 09:53:49
12 min ago
-
Breaking news - Update 1 - 12/09/2023 09:45:16
21 min ago
-
Breaking news - Update 1 - 12/09/2023 09:41:05
22 min ago
-
Breaking news - Update 1 - 12/09/2023 09:37:54
28 min ago
-
Breaking news - Update 1 - 12/09/2023 09:33:58
29 min ago
-
dsgff
29 min ago
-
Breaking news - Update 1 - 12/09/2023 09:29:41
36 min ago