diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average.json" @@ -0,0 +1,33680 @@ +{ + "original_ppl": 3.697141408920288, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3464.481201171875 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3677.32177734375 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3220.70751953125 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 2131.33447265625 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3050.191650390625 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3169.390380859375 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 2524.9228515625 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3505.794189453125 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3512.329833984375 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3000.78466796875 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3446.185791015625 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3538.577880859375 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3145.047119140625 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3348.456787109375 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3593.623779296875 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3501.1298828125 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3036.5185546875 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3866.78857421875 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3348.3193359375 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3167.444580078125 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3323.9296875 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3516.482421875 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3708.910400390625 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4347.1240234375 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3240.3759765625 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3352.1630859375 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3381.581298828125 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3230.160400390625 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3474.3310546875 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3624.54541015625 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3904.429443359375 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3931.96875 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3287.423583984375 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3556.094970703125 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3329.392578125 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3602.624267578125 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3520.4921875 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3685.15771484375 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3495.671630859375 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3414.18310546875 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3327.97998046875 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3443.551025390625 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3444.01416015625 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3339.546142578125 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 2957.610595703125 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3380.78125 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3418.731689453125 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3345.331787109375 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3653.060302734375 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3615.57275390625 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3484.28857421875 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3334.352783203125 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3082.19921875 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3499.62109375 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3387.6494140625 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3406.943115234375 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3531.309814453125 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3506.603515625 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3868.710205078125 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3764.70654296875 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3482.587890625 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3693.3271484375 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3579.640869140625 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3575.9287109375 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3613.32177734375 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3786.64208984375 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3601.834228515625 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3556.9462890625 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3534.129638671875 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3533.364501953125 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3586.906005859375 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3581.355224609375 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3393.70947265625 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3100.755615234375 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3498.159423828125 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3489.486328125 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3524.822509765625 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3541.474609375 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3562.73486328125 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3857.061279296875 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3233.695556640625 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3500.315185546875 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3616.383056640625 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3580.91796875 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3646.1826171875 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3648.7255859375 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3388.560546875 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3332.454833984375 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3620.230712890625 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3646.46435546875 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3618.018310546875 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3555.70849609375 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3578.958251953125 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3403.302734375 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3548.899169921875 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3505.10205078125 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3525.427734375 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3479.354736328125 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3556.078125 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3519.56591796875 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3695.744140625 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3425.1474609375 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3614.25927734375 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3449.5693359375 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3472.6650390625 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3519.465087890625 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3514.176025390625 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3397.32958984375 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3332.216796875 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3525.00732421875 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3529.767578125 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3414.82470703125 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3572.935546875 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3578.476806640625 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3620.157958984375 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3347.22412109375 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3481.14013671875 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3397.452880859375 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3501.714111328125 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3550.26318359375 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3554.250732421875 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3657.511962890625 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3503.755126953125 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3498.2060546875 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3591.5576171875 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3502.08154296875 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3557.892822265625 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3562.761962890625 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3387.365234375 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3597.06640625 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3500.82275390625 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3503.26416015625 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3435.848876953125 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3592.839111328125 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3547.57958984375 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3612.81884765625 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3530.255615234375 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3581.252685546875 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3535.090087890625 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3484.102783203125 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3616.645263671875 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3582.776123046875 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3615.041748046875 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3424.03369140625 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3534.27783203125 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3601.5146484375 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3554.345703125 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3570.53076171875 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3552.1259765625 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3440.91162109375 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3528.848876953125 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3523.068359375 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3497.592041015625 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3476.7373046875 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3579.920654296875 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3554.05078125 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3449.1318359375 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3632.650146484375 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3574.29541015625 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3606.746337890625 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3529.7841796875 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3531.518310546875 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3541.02197265625 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3489.4365234375 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3556.959716796875 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3601.68994140625 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3603.177734375 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3553.97265625 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3526.6650390625 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3569.91455078125 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3655.628662109375 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3565.99462890625 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3552.451171875 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3576.16064453125 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3590.4892578125 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3606.357666015625 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3585.305419921875 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3383.326171875 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3453.83544921875 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3570.1494140625 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3518.126220703125 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3598.716796875 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3585.339599609375 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3540.32666015625 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3647.01416015625 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3611.468505859375 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3496.431640625 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3465.0263671875 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3481.375732421875 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3568.396240234375 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3558.585205078125 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3567.113525390625 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3462.037109375 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3537.369873046875 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3454.96240234375 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3482.454833984375 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3580.498046875 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3562.57177734375 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3555.57958984375 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3659.325927734375 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3480.834716796875 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3484.60107421875 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3482.33544921875 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3526.134033203125 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3547.802978515625 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3615.197021484375 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3383.19384765625 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3525.115234375 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3534.8408203125 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3586.694091796875 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3581.105712890625 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3566.310791015625 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3430.217529296875 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3412.60107421875 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3544.95166015625 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3576.293701171875 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3588.49365234375 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3560.6220703125 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3558.55126953125 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3364.076416015625 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3563.0439453125 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3550.591796875 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3594.59716796875 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3594.569580078125 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3594.50146484375 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3576.072021484375 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3421.145263671875 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3577.722900390625 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3514.330322265625 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3542.4677734375 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3514.33349609375 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3542.069091796875 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3515.218505859375 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3684.802490234375 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3762.018310546875 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3573.9306640625 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3572.294921875 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3504.58056640625 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3569.55712890625 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3580.8291015625 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3705.78515625 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3521.28466796875 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3599.454833984375 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3466.814453125 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3495.891357421875 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3594.898681640625 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3569.083984375 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3679.317626953125 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3586.215087890625 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3479.630126953125 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3414.84423828125 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3582.878662109375 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3625.776123046875 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3583.28857421875 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3446.153076171875 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3570.9599609375 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3478.093994140625 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3520.408447265625 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3611.344482421875 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3527.7685546875 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3514.8330078125 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3553.281494140625 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3497.55224609375 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3538.7939453125 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3457.945556640625 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3492.0498046875 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3582.2705078125 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3596.208740234375 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3400.165771484375 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3518.549072265625 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3572.260986328125 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3459.627685546875 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3417.1669921875 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3565.590087890625 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3547.9111328125 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3573.610595703125 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3516.264404296875 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3507.37255859375 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3456.105712890625 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3545.1376953125 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3559.121337890625 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3523.32373046875 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3329.42138671875 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3736.813232421875 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3528.31689453125 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3544.701416015625 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3549.63037109375 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3636.865234375 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3560.995361328125 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3538.18994140625 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3502.953369140625 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3496.99853515625 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3557.852294921875 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3510.022705078125 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3564.0771484375 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3545.306640625 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3403.52978515625 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3596.146728515625 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3488.4248046875 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3589.684814453125 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3531.50146484375 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3578.11181640625 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3580.8837890625 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3546.92333984375 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3489.386474609375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3459.037109375 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3493.471923828125 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3558.7548828125 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3516.304931640625 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3520.086181640625 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3563.96826171875 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3482.913330078125 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3543.123046875 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3548.310302734375 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3490.035400390625 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3539.091064453125 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3532.9736328125 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3556.837890625 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3466.3583984375 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3571.3857421875 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3528.11181640625 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3523.404296875 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3550.706787109375 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3552.03125 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3618.64306640625 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3700.209228515625 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3508.275634765625 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3543.61328125 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3558.97900390625 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3557.580810546875 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3568.529052734375 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3550.137939453125 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3618.56689453125 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3565.09375 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3551.69580078125 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3554.857421875 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3553.267822265625 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3556.234130859375 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3574.789794921875 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3567.1953125 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3521.724609375 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3572.42822265625 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3505.3828125 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3557.526611328125 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3562.70751953125 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3523.824462890625 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3506.0615234375 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3544.02880859375 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3531.734130859375 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3566.7734375 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3565.3994140625 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3555.77294921875 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3636.140380859375 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3626.523193359375 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3547.261474609375 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3565.2294921875 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3543.94775390625 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3546.78466796875 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3545.637939453125 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3511.783935546875 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3548.662109375 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3539.570068359375 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3569.689697265625 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3555.94921875 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3569.839599609375 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3568.117431640625 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3489.090087890625 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3524.620849609375 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3515.684326171875 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3541.211181640625 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3559.54248046875 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3559.8037109375 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3552.529296875 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3529.31298828125 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3501.303466796875 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3544.8671875 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3548.6689453125 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3551.418212890625 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3552.16015625 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3551.309814453125 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3567.399658203125 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3499.06689453125 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3537.140380859375 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3532.791748046875 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3530.0234375 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3560.214599609375 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3543.17724609375 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3515.684326171875 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3533.792724609375 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3533.68798828125 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3562.948974609375 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3537.25537109375 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3569.530029296875 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3553.112060546875 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3495.90478515625 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3468.282958984375 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3518.354248046875 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3540.329833984375 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3532.724365234375 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3551.790771484375 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3559.3115234375 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3564.862548828125 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3495.2314453125 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3538.206787109375 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3534.928466796875 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3577.497802734375 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3562.66015625 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3554.98291015625 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3525.93212890625 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3551.787353515625 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3512.072021484375 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3554.606689453125 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3531.20849609375 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3566.161376953125 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3554.548828125 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3594.090087890625 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3638.391357421875 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3517.961669921875 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3524.392333984375 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3545.21875 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3570.564697265625 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3556.678466796875 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3582.2021484375 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3548.87890625 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3518.823974609375 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3520.982421875 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3587.408935546875 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3551.90576171875 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3552.94580078125 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3533.260009765625 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3465.793212890625 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3572.6630859375 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3570.96337890625 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3549.400146484375 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3550.20263671875 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3552.08544921875 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3561.250244140625 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3524.570556640625 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3564.461181640625 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3547.227783203125 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3566.998046875 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3563.434814453125 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3560.856201171875 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3517.49560546875 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3594.71728515625 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3570.108642578125 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3522.974365234375 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3528.97314453125 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3553.93212890625 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3550.4091796875 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3491.064208984375 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3587.966552734375 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3556.359619140625 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3566.749755859375 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3528.774658203125 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3555.28466796875 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3557.7333984375 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3516.355224609375 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3582.998291015625 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3549.5625 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3512.741943359375 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3550.351318359375 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3539.79638671875 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3551.49951171875 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3550.679931640625 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3606.00341796875 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3535.595947265625 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3560.23486328125 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3591.150390625 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3569.308837890625 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3568.093505859375 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3533.839599609375 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3567.950439453125 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3572.162109375 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3518.13623046875 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3523.737060546875 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3541.10302734375 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3543.457763671875 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3527.93017578125 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3612.70166015625 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3546.033447265625 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3601.41845703125 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3519.5390625 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3547.81640625 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3554.748779296875 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3654.381103515625 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3527.40185546875 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3526.718994140625 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3566.67822265625 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3479.480712890625 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3577.4873046875 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3581.877685546875 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3606.828857421875 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3558.83642578125 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3533.543212890625 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3586.919921875 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3466.576416015625 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3552.190673828125 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3544.238525390625 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3628.2978515625 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3480.3369140625 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3502.919921875 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3559.270751953125 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3519.834228515625 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3542.278564453125 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3545.928955078125 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3588.63037109375 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3549.474609375 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3498.77001953125 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3538.935791015625 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3528.431396484375 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3553.376220703125 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3546.984130859375 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3502.365478515625 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3529.730712890625 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3485.910888671875 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3542.099365234375 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3547.71484375 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3530.568603515625 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3592.58203125 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3440.7705078125 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3562.411865234375 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3584.35498046875 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3511.000244140625 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3602.765380859375 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3510.27392578125 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3519.12255859375 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3594.0078125 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3542.25146484375 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3545.79345703125 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3633.08642578125 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3593.10302734375 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3584.33447265625 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3594.144775390625 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3612.37109375 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3576.371826171875 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3455.83544921875 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3628.678466796875 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3574.20361328125 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3504.771240234375 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3512.302978515625 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3581.53955078125 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3587.593505859375 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3645.18505859375 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3749.11376953125 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3590.612548828125 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3561.281005859375 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3498.61962890625 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3530.710205078125 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3567.9404296875 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3521.49609375 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3712.47412109375 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3666.239013671875 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3564.026123046875 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3569.373291015625 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3541.224609375 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3621.1181640625 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3426.92822265625 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3298.002685546875 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3188.660400390625 + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.587299823760986 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.621645927429199 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.606184482574463 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.591529846191406 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.595300197601318 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.606298923492432 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.578024864196777 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.558347225189209 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.565422534942627 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.539042949676514 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.525199890136719 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.55272912979126 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.567893028259277 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.547338962554932 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.578011989593506 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.579082012176514 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.542652130126953 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.5580949783325195 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.531676769256592 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.560765743255615 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.541767120361328 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.579802989959717 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.601284503936768 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.570128917694092 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.569732189178467 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.539124488830566 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.561048984527588 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.572926998138428 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.5919389724731445 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.621241569519043 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.600897789001465 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.587358474731445 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.54921817779541 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.582107067108154 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.573081970214844 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.612077713012695 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.620278835296631 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.593531608581543 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.605550289154053 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.571804523468018 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.580696105957031 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.580711841583252 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.605426788330078 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.603277206420898 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.587345123291016 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.597412109375 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.5576558113098145 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.583559989929199 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.582685947418213 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.619879245758057 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.603657245635986 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.585909843444824 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.600263595581055 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.580042362213135 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.583291530609131 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.5865478515625 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.607184886932373 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.629411220550537 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.588752746582031 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.598271369934082 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.586933612823486 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.582657814025879 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.591380596160889 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.608002185821533 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.605777740478516 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.598511695861816 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.586959362030029 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.582998275756836 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.595479488372803 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.591337203979492 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.6023688316345215 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.600842475891113 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.5933027267456055 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.587803840637207 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.579985618591309 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.587390422821045 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.576834201812744 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.60318660736084 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.603156089782715 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.598178863525391 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.595784664154053 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.581279754638672 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.592929363250732 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.599449157714844 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.607228755950928 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.60359001159668 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.591875076293945 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.5998334884643555 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.584235191345215 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.582729339599609 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.591054916381836 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.602621555328369 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.607583045959473 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.578174591064453 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.580382347106934 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.600681781768799 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.601696968078613 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.586922645568848 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.600650787353516 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.599786281585693 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.593698024749756 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.588168144226074 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.5824480056762695 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.58252477645874 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.589319705963135 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.601154327392578 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.602333068847656 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.596490859985352 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.599719524383545 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.591432571411133 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.603636264801025 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.603952407836914 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.6040825843811035 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.601742267608643 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.602610111236572 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.602569103240967 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.585309982299805 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.603806495666504 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.587503910064697 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.593698024749756 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.5925798416137695 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.5892462730407715 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.591400623321533 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.60642671585083 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.597780704498291 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.601413726806641 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.59487247467041 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.600255012512207 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.594669818878174 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.588927268981934 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.597878932952881 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.606900215148926 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.590006351470947 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.594246864318848 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.596985340118408 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.601166725158691 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.593268871307373 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.598250865936279 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.605244159698486 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.604512691497803 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.587610244750977 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.601518630981445 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.6021952629089355 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.605716705322266 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.603775501251221 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.607298374176025 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.6122212409973145 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.593820571899414 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.599548816680908 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.596066474914551 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.593171119689941 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.599617004394531 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.605527400970459 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.602389812469482 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.598952293395996 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.602376937866211 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.606771945953369 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.602109432220459 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.592378616333008 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.590651512145996 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.6001057624816895 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.598791122436523 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.604039669036865 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.604513168334961 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.603310585021973 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.588095188140869 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.601797580718994 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.600030422210693 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.606400489807129 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.600368499755859 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.597190856933594 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.589382648468018 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.587274074554443 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.596391677856445 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.59694242477417 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.592281341552734 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.59852933883667 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.57909631729126 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.581469535827637 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.600376129150391 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.596221923828125 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.605307579040527 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.600579738616943 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.60133171081543 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.605119705200195 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.60227108001709 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.598724365234375 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.606266975402832 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.598618507385254 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.598852157592773 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.601508140563965 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.598553657531738 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.601412296295166 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.59518575668335 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.594697952270508 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.597851276397705 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.6041178703308105 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.600735187530518 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.601080894470215 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.60016393661499 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.5956621170043945 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.600144863128662 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.597470760345459 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.6035685539245605 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.603593826293945 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.605932712554932 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.599174976348877 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.595711708068848 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.600107669830322 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.5986552238464355 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.601630210876465 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.602270603179932 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.6023478507995605 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.600612163543701 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.587116241455078 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.594443321228027 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.597299575805664 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.605952262878418 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.60903263092041 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.610475540161133 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.610025405883789 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.594229221343994 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.595969200134277 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.597070693969727 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 6.599231719970703 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 6.601463794708252 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 6.602692127227783 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 6.603835105895996 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 6.584806442260742 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 6.598879814147949 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 6.592294216156006 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 6.599766254425049 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 6.602532386779785 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 6.606451988220215 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 6.607882022857666 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 6.596158981323242 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 6.59637451171875 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 6.607106685638428 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 6.607349872589111 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 6.609874248504639 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 6.612264156341553 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 6.604864597320557 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 6.598066806793213 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 6.592206954956055 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 6.596440315246582 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 6.604358196258545 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 6.605189800262451 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 6.613621234893799 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 6.606575965881348 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 6.6003241539001465 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 6.607431888580322 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 6.603500843048096 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 6.61030387878418 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 6.605193614959717 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 6.612126350402832 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 6.603789806365967 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 6.59227991104126 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 6.593422889709473 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 6.596105575561523 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 6.604356288909912 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 6.607855796813965 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 6.598106384277344 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 6.600514888763428 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 6.605868339538574 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 6.606652736663818 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 6.601559162139893 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 6.599547386169434 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 6.603506088256836 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 6.5987725257873535 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 6.601705074310303 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 6.595846176147461 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 6.605261325836182 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 6.6010050773620605 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 6.605607986450195 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 6.607065200805664 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 6.599732875823975 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 6.601710319519043 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 6.601534843444824 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 6.6002373695373535 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 6.601970672607422 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 6.599880218505859 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 6.599278926849365 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 6.596271514892578 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 6.596656799316406 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 6.596795558929443 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 6.602492809295654 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 6.600967884063721 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 6.599152565002441 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 6.597898483276367 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 6.594899654388428 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 6.597871780395508 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 6.596582412719727 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 6.601064205169678 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 6.595006465911865 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 6.603090286254883 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 6.601056098937988 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 6.60007381439209 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 6.606298923492432 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 6.59986686706543 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 6.601884365081787 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 6.59735631942749 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 6.601327896118164 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 6.606303691864014 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 6.604212760925293 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 6.601311206817627 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 6.597829341888428 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 6.6004838943481445 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 6.598099231719971 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 6.608369827270508 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 6.606858730316162 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 6.600128650665283 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 6.6029767990112305 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 6.6003546714782715 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 6.59958028793335 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 6.593913555145264 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 6.604043006896973 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 6.604269027709961 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 6.606225967407227 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 6.606950283050537 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 6.595943450927734 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 6.599369525909424 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 6.598005294799805 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 6.603532791137695 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 6.607446193695068 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 6.605018138885498 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 6.60512638092041 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 6.598582744598389 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 6.5997185707092285 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 6.5999531745910645 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 6.6023406982421875 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 6.602704048156738 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 6.601614952087402 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 6.599346160888672 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 6.5980119705200195 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 6.596429347991943 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 6.596161365509033 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 6.605160236358643 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 6.611623764038086 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 6.610204219818115 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 6.608973026275635 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 6.599440574645996 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 6.597598075866699 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 6.597370624542236 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 6.605222225189209 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 6.604358196258545 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 6.602878570556641 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 6.605030536651611 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 6.5970940589904785 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 6.600498676300049 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 6.596414566040039 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 6.6046833992004395 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 6.6062541007995605 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 6.605898857116699 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 6.607046604156494 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 6.597235679626465 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 6.599125862121582 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 6.597239017486572 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 6.603255748748779 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 6.604758262634277 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 6.60517692565918 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 6.607698440551758 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 6.5967912673950195 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 6.599171161651611 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 6.596045017242432 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 6.606232643127441 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 6.608950138092041 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 6.60835599899292 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 6.607090950012207 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 6.6011457443237305 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 6.600656509399414 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 6.596714019775391 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 6.604354381561279 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 6.605943202972412 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 6.6068572998046875 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 6.608158111572266 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 6.598452568054199 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 6.5994553565979 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 6.598047733306885 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 6.603742599487305 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 6.6061906814575195 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 6.606211185455322 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 6.604738712310791 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 6.5995354652404785 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 6.599943161010742 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 6.5973687171936035 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 6.604788780212402 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 6.605924606323242 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 6.60308837890625 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 6.603743553161621 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 6.598557472229004 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 6.598172664642334 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 6.6009602546691895 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 6.604080677032471 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 6.605562210083008 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 6.605836391448975 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 6.603350639343262 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 6.598554611206055 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 6.6019134521484375 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 6.597433090209961 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 6.606156349182129 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 6.606710433959961 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 6.607789516448975 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 6.6072096824646 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 6.602711200714111 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 6.600829601287842 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 6.599113941192627 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 6.6030168533325195 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 6.602009296417236 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 6.60439920425415 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 6.603659629821777 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 6.598233699798584 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 6.5990753173828125 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 6.597658634185791 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 6.6041483879089355 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 6.604606628417969 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 6.604905605316162 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 6.603374481201172 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 6.599079132080078 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 6.5978217124938965 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 6.6002726554870605 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 6.604966640472412 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 6.604852199554443 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 6.6043243408203125 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 6.604616641998291 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 6.599572658538818 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 6.601479530334473 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 6.600064754486084 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 6.604489803314209 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 6.604806423187256 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 6.606635093688965 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 6.60684871673584 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 6.602787017822266 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 6.6002655029296875 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 6.60007905960083 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 6.6053385734558105 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 6.604893207550049 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 6.605632305145264 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 6.604212760925293 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 6.600472450256348 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 6.59916353225708 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 6.600151538848877 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 6.6047563552856445 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 6.605164051055908 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 6.60540246963501 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 6.605785846710205 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 6.597067356109619 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 6.599972248077393 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 6.600137233734131 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 6.604395389556885 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 6.6043524742126465 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 6.604781150817871 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 6.605875492095947 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 6.599791049957275 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 6.601497173309326 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 6.599152565002441 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 6.603846073150635 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 6.604429721832275 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 6.6054158210754395 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 6.604636192321777 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 6.599225997924805 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 6.6016764640808105 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 6.600290298461914 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 6.604108810424805 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 6.604352951049805 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 6.604194641113281 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 6.605434417724609 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 6.599965572357178 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 6.600320339202881 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 6.600098133087158 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 6.603488445281982 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 6.604409694671631 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 6.605992317199707 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 6.605588912963867 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 6.5988593101501465 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 6.599915504455566 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 6.602593898773193 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 6.603749752044678 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 6.603925704956055 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 6.603025913238525 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 6.603877544403076 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 6.600656509399414 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 6.601109981536865 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 6.597596168518066 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 6.605644226074219 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 6.605431079864502 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 6.603664875030518 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 6.604753017425537 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 6.60113525390625 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 6.6016106605529785 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 6.600804328918457 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 6.604508876800537 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 6.606046676635742 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 6.603638648986816 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 6.607027053833008 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 6.5996527671813965 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 6.599191665649414 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 6.600780487060547 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 6.603200435638428 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 6.603689670562744 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 6.599296569824219 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 6.601716041564941 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 6.599742889404297 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 6.598677158355713 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 6.601217746734619 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 6.6124982833862305 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 6.612306118011475 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 6.606039524078369 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 6.602809906005859 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 6.601545333862305 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 6.604678630828857 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 6.602466583251953 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 6.603470802307129 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 6.603786945343018 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 6.604178428649902 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 6.602931022644043 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 6.599786281585693 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 6.602194309234619 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 6.6041646003723145 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 6.619690418243408 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 6.620758533477783 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 6.615204811096191 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 6.61479377746582 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 6.595831871032715 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 6.594951629638672 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 6.601993083953857 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 6.605774402618408 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 6.605727672576904 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 6.606330871582031 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 6.608585357666016 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 6.6034440994262695 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 6.605523109436035 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 6.60240364074707 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 6.633901596069336 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 6.626533508300781 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 6.61317253112793 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 6.613747596740723 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 6.583765506744385 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 6.603389263153076 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 6.60793924331665 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 6.605195999145508 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 6.6029372215271 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 6.603430271148682 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 6.605612277984619 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 6.606897830963135 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 6.604814052581787 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 6.606776714324951 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 6.604368686676025 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 6.604959011077881 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 6.617969036102295 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 6.60932731628418 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 6.589051246643066 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 6.588523864746094 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 6.589207649230957 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 6.603640079498291 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 6.6061506271362305 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 6.60843563079834 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 6.607334136962891 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 6.621618747711182 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 6.6069841384887695 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 6.545148849487305 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.370658874511719 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.3709917068481445 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.372006416320801 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.370563507080078 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.37232780456543 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.373668670654297 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.3709635734558105 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.367496013641357 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.368441581726074 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.367706775665283 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.365130424499512 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.367547988891602 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.369000434875488 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.367656230926514 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.368114948272705 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.369100570678711 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.367117881774902 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.366388320922852 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.36487340927124 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.368755340576172 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.365574836730957 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.368518352508545 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.369019508361816 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.369363307952881 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.368993282318115 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.3654866218566895 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.367645263671875 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.368037223815918 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.369693279266357 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.369786262512207 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.369330883026123 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.36944580078125 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.366352081298828 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.368671894073486 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.368444442749023 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.371212482452393 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.372028827667236 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.370617389678955 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.370952606201172 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.366947174072266 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.368783473968506 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.369964122772217 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.3701066970825195 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.370917320251465 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.37069034576416 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.369626522064209 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.367837429046631 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.3685102462768555 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.367172718048096 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.371077060699463 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.370008945465088 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.370212078094482 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.370441436767578 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.369170188903809 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.366943836212158 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.369058132171631 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.370612621307373 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.371523857116699 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.369246959686279 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.367514133453369 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.367551326751709 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.366318702697754 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.368446350097656 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.370789051055908 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.3708600997924805 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.370893955230713 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.370158672332764 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.367159843444824 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.370361804962158 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.367939472198486 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.37029504776001 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.370350360870361 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.369771480560303 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.369573593139648 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.366819381713867 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.3677144050598145 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.368299961090088 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.37019157409668 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.371087074279785 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.370404243469238 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.3695831298828125 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.366743564605713 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.369704723358154 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.36887264251709 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.370004653930664 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.370041847229004 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.370377063751221 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.371026992797852 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.36906623840332 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.370051383972168 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.369397163391113 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.370617389678955 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.3704142570495605 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.368880748748779 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.36940336227417 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.369184970855713 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.369528770446777 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.3685126304626465 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.369605541229248 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.369452476501465 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.370082855224609 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.369543075561523 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.36824893951416 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.368817329406738 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.370004177093506 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.371281147003174 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.3708906173706055 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.370126724243164 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.369634628295898 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.368972301483154 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.37039041519165 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.369751453399658 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.370290279388428 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.370081424713135 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.370199680328369 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.369494438171387 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.36939001083374 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.369959354400635 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.36924409866333 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.369386196136475 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.3701677322387695 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.368776798248291 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.370244979858398 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.370024681091309 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.371572971343994 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.369118690490723 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.369749546051025 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.36985969543457 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.36874532699585 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.36946439743042 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.369516372680664 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.369718551635742 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.370192527770996 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.37045431137085 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.370177268981934 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.3698554039001465 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.36964750289917 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.368263244628906 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.370279312133789 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.370323657989502 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.3698954582214355 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.370196342468262 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.36991024017334 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.369865417480469 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.369717121124268 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.368814468383789 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.368879318237305 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.369174480438232 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.369974613189697 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.369632720947266 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.369807720184326 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.369532585144043 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.369607448577881 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.370362281799316 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.370495319366455 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.370677947998047 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.370682716369629 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.370794296264648 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.369177341461182 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.3691606521606445 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.369308948516846 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.369301795959473 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.370370388031006 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.370275020599365 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.369775772094727 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.369058132171631 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.369899272918701 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.369477272033691 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.3701043128967285 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.370334148406982 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.370373725891113 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.370621681213379 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.368663311004639 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.369245529174805 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.370733261108398 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.369558811187744 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.37009859085083 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.3681817054748535 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.367650032043457 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.369901657104492 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.369140148162842 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.370650768280029 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.37069034576416 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.370615482330322 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.370699405670166 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.370719909667969 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.369001865386963 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.369406700134277 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.370663166046143 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.369678497314453 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.369998455047607 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.37005615234375 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.369414806365967 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.369797229766846 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.369734764099121 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.369627952575684 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.370367527008057 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.370331764221191 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.370481491088867 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.369726657867432 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.369414806365967 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.369966506958008 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.368894577026367 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.370353698730469 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.3705620765686035 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.370404243469238 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.369851589202881 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.369944095611572 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.36950159072876 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.37017297744751 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.370259761810303 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.370347499847412 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.370332717895508 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.370377540588379 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.369397163391113 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.369746685028076 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.368403911590576 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.369879722595215 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.370453834533691 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.3704681396484375 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.3705668449401855 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.369260787963867 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.3699421882629395 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.368878364562988 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.370258331298828 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.370457172393799 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.3696722984313965 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.3701300621032715 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.367971420288086 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.369325637817383 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.36854887008667 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.370546817779541 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.370273113250732 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.369988918304443 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.369546413421631 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.367751598358154 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.369838237762451 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.369894027709961 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.370205879211426 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.370344638824463 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.369682312011719 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.369753360748291 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.369647026062012 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.369004726409912 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.370223522186279 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.370478630065918 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.370723724365234 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.36993408203125 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.370380878448486 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.369374752044678 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.370112419128418 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.369381427764893 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.370210647583008 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.370704174041748 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.370889186859131 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.370656967163086 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.36891508102417 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.370216369628906 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.36879825592041 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.370626926422119 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.3706440925598145 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.370516300201416 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.370355129241943 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.369748592376709 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.370649814605713 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.37078332901001 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.370316982269287 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.370417594909668 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.369919776916504 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.3699798583984375 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.3686203956604 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.369569301605225 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.3697309494018555 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.369585990905762 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.370404243469238 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.370459079742432 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.369085311889648 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.369277000427246 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.370120525360107 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.3698225021362305 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.369800567626953 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.369987964630127 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.369215965270996 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.369477272033691 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.369507789611816 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.3702311515808105 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.369599342346191 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.369622230529785 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.369960784912109 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.369366645812988 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.370115280151367 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.368876934051514 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.369987964630127 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.3692946434021 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.371050834655762 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.370308876037598 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.370563507080078 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.370288372039795 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.368695259094238 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.370031833648682 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.37069845199585 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.369816780090332 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.370055198669434 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.369913101196289 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.369883060455322 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.369180679321289 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.3699164390563965 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.369873523712158 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.370911598205566 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.370357036590576 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.3701300621032715 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.369647979736328 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.36823844909668 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.368480682373047 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.368778228759766 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.370745658874512 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.370642185211182 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.370589256286621 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.370792388916016 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.369303226470947 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.369446754455566 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.368327617645264 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.370411396026611 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.370616436004639 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.370595932006836 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.370485305786133 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.369943141937256 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.369462490081787 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.370121002197266 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.3701372146606445 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.370247840881348 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.370376110076904 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.369966506958008 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.3690266609191895 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.3697829246521 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.369048595428467 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.370481014251709 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.370688438415527 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.370548725128174 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.370810508728027 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.368679523468018 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.369833469390869 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.3692827224731445 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.370330333709717 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.370555877685547 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.37021541595459 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.37031364440918 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.368698596954346 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.369692325592041 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.3687238693237305 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.370598793029785 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.3705058097839355 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.370670795440674 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.370545387268066 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.369062900543213 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.369428634643555 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.369572639465332 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.370123386383057 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.370323657989502 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.37005615234375 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.370494842529297 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.368701934814453 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.370166301727295 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.368327617645264 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.3705291748046875 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.370993614196777 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.370922088623047 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.370280742645264 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.368600845336914 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.369245529174805 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.368486404418945 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.3705644607543945 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.370523452758789 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.3705854415893555 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.370707035064697 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.368817329406738 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.369629383087158 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.36944055557251 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.370484352111816 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.370792388916016 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.370545864105225 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.370151519775391 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.369105815887451 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.369824409484863 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.36972188949585 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.370765686035156 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.370667457580566 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.370337009429932 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.37004280090332 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.369317054748535 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.369020462036133 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.3690080642700195 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.370368957519531 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.370558261871338 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.370368957519531 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.370271682739258 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.367918014526367 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.369907855987549 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.368680953979492 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.370500087738037 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.37069845199585 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.370538234710693 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.370504379272461 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.368698596954346 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.370347023010254 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.369478702545166 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.370362281799316 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.369499683380127 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.370457172393799 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.370245933532715 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.369790554046631 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.369180679321289 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.3686676025390625 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.370546817779541 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.3705244064331055 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.370411396026611 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.370611190795898 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.368659973144531 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.370439529418945 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.368627548217773 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.370355129241943 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.3705339431762695 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.370579719543457 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.370670318603516 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.368842124938965 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.368569374084473 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.368898391723633 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.370463848114014 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.370635986328125 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.370703220367432 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.370877265930176 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.369894027709961 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.369246482849121 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.368661880493164 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.370605945587158 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.370710372924805 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.370368957519531 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.370603084564209 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.368288993835449 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.368137359619141 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.369358539581299 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.370471000671387 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.370543479919434 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.3706560134887695 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.3705315589904785 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.367757320404053 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.369256019592285 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.368664741516113 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.370599746704102 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.370527267456055 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.370788097381592 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.370481967926025 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.3685994148254395 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.369345188140869 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.3690643310546875 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.370444297790527 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.370588302612305 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.370532989501953 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.3704423904418945 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.368338108062744 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.369311332702637 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.3697028160095215 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.370581150054932 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.370548248291016 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.370380878448486 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.370533466339111 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.368878364562988 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.369256019592285 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.369171142578125 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.370511531829834 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.370306015014648 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.372114658355713 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.37081241607666 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.368305683135986 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.369886875152588 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.369573593139648 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.370284080505371 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.370397567749023 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.370340824127197 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.3701066970825195 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.3685431480407715 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.369591236114502 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.369800090789795 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.370703220367432 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.370458126068115 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.3702239990234375 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.370509147644043 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.369507789611816 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.36972713470459 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.3690361976623535 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.370020389556885 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.369931221008301 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.370157241821289 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.370376110076904 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.3678412437438965 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.369663238525391 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.370247840881348 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.370338439941406 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.370195388793945 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.370934009552002 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.370158672332764 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.368406772613525 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.369710922241211 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.3704047203063965 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.37039852142334 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.370025157928467 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.370766639709473 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.370181083679199 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.368619918823242 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.369656085968018 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.369997024536133 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.370120525360107 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.370104789733887 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.370178699493408 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.370549201965332 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.369039058685303 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.369682788848877 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.370556354522705 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.3708176612854 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.371338844299316 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.3713698387146 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.370495319366455 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.367344379425049 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.370899677276611 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.369388103485107 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.3706560134887695 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.370896816253662 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.370218753814697 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.370737552642822 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.366901397705078 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.369959831237793 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.370751857757568 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.372854232788086 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.37371826171875 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.372132301330566 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.371866226196289 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.368535995483398 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.370369911193848 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.370703220367432 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.370266437530518 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.370609283447266 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.3709564208984375 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.370542526245117 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.367404937744141 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.371592998504639 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.369471549987793 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.370596408843994 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.370932102203369 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.371057510375977 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.370301723480225 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.365617752075195 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.367737293243408 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.370293617248535 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.370628833770752 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.3704962730407715 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.370964050292969 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.370872974395752 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.37193489074707 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.3698039054870605 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.364621639251709 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.989259958267212 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.989586591720581 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.989457607269287 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.989313840866089 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.9897987842559814 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.9898786544799805 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.990248203277588 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.9888205528259277 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.9888482093811035 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.988792657852173 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.988403797149658 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.9887535572052 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.9891655445098877 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.9890999794006348 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.988828659057617 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.9891767501831055 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.9891536235809326 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.9891157150268555 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.9883594512939453 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.9891726970672607 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.9888460636138916 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.9889020919799805 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.9890296459198 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.9892048835754395 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.9892704486846924 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.988804340362549 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.9889495372772217 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.988839864730835 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.989001750946045 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.9892921447753906 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.9890191555023193 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.989163875579834 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.9881536960601807 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.9889028072357178 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.988525152206421 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.989436149597168 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.98948335647583 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.989192247390747 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.9888899326324463 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.9885690212249756 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.989027738571167 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.9890570640563965 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.9894745349884033 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.9904227256774902 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.9890286922454834 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.9892079830169678 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.988530397415161 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.9888336658477783 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.9892683029174805 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.98945951461792 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.9891886711120605 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.9894843101501465 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.9893927574157715 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.9892241954803467 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.9896702766418457 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.988649368286133 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.9897477626800537 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.9896512031555176 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.9891316890716553 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.9886972904205322 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.9881975650787354 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.988278865814209 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.988563060760498 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.9895379543304443 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.9894497394561768 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.9892513751983643 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.989354372024536 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.988262891769409 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.9891200065612793 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.989325523376465 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.9892842769622803 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.9892308712005615 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.9894282817840576 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.9892523288726807 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.987931251525879 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.9885494709014893 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.9885199069976807 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.9891483783721924 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.989225149154663 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.989548921585083 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.988978385925293 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.989313840866089 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.9890308380126953 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.9890964031219482 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.9892144203186035 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.9892191886901855 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.9894723892211914 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.9888992309570312 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.9884533882141113 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.988980293273926 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.988579750061035 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.9894180297851562 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.9894254207611084 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.9893949031829834 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.988919258117676 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.988879680633545 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.989042043685913 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.9887733459472656 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.9894254207611084 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.989112377166748 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.9888577461242676 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.9885380268096924 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.9889726638793945 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.9891836643218994 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.9896981716156006 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.9893062114715576 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.989243745803833 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.9898037910461426 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.988999128341675 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.989009380340576 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.9892008304595947 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.9893577098846436 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.989319086074829 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.9888699054718018 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.9891715049743652 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.9891586303710938 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.9890296459198 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.9885408878326416 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.9893174171447754 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.9891855716705322 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.9888226985931396 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.988914728164673 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.9891855716705322 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.989671468734741 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.9894156455993652 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.9892466068267822 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.9891998767852783 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.988955020904541 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.9889097213745117 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.9881837368011475 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.988870143890381 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.988237142562866 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.989407777786255 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.9892232418060303 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.988959312438965 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.9897563457489014 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.988868474960327 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.9888534545898438 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.988537073135376 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.989206552505493 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.989356756210327 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.9894490242004395 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.9891295433044434 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.9892303943634033 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.989528179168701 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.9892919063568115 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.989149332046509 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.9892704486846924 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.9888741970062256 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.98921799659729 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.988981246948242 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.9894840717315674 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.98880672454834 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.989325523376465 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.9894766807556152 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.989553928375244 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.989331007003784 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.9888322353363037 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.9893970489501953 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.989107608795166 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.989248275756836 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.9893062114715576 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.989201784133911 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.9893617630004883 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.9887330532073975 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.988969326019287 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.9886586666107178 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.9891247749328613 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.9890761375427246 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.989189624786377 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.9892144203186035 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.9881975650787354 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.9889137744903564 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.9891319274902344 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.9891629219055176 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.9892027378082275 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.9889299869537354 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.988969326019287 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.9887330532073975 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.9891817569732666 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.988999128341675 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.9893929958343506 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.989374876022339 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.9892189502716064 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.9891698360443115 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.988983631134033 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.9890239238739014 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.9891390800476074 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.9892122745513916 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.988948345184326 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.9888322353363037 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.9887423515319824 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.9891672134399414 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.9893393516540527 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.9892256259918213 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.989337921142578 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.9892892837524414 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.989330291748047 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.989302635192871 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.989403009414673 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.9891579151153564 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.9892680644989014 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.989086866378784 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.9888813495635986 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.9889028072357178 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.989297866821289 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.988936424255371 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.9893620014190674 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.989352226257324 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.9891886711120605 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.9892687797546387 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.9893035888671875 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.9888899326324463 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.9887938499450684 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.9891417026519775 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.9893686771392822 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.9892961978912354 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.988906145095825 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.9887096881866455 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.9892475605010986 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.989457607269287 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.989461660385132 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.9895946979522705 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.989490509033203 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.9885165691375732 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.9887421131134033 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.989222526550293 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.989403486251831 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.9890494346618652 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.9894766807556152 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.988926887512207 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.988981246948242 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.989095687866211 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.9890990257263184 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.9892289638519287 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.988813877105713 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.9888675212860107 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.9890201091766357 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.989264488220215 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.9891629219055176 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.9892687797546387 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.9892332553863525 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.9892725944519043 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.9892780780792236 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.988875150680542 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.9892826080322266 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.9892654418945312 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.9891510009765625 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.9892423152923584 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.9888503551483154 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.989189624786377 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.9888744354248047 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.988682746887207 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.988732099533081 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.989354372024536 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.989346981048584 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.989440441131592 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.9893927574157715 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.988666534423828 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.9890897274017334 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.989128589630127 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.9891953468322754 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.9893593788146973 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.9888129234313965 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.9893574714660645 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.9888243675231934 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.9891157150268555 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.9889512062072754 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.988999843597412 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.989187717437744 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.9890007972717285 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.989269495010376 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.9888978004455566 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.9890239238739014 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.989375591278076 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.9889957904815674 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.989133834838867 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.9887492656707764 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.9885671138763428 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.9892354011535645 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.9889469146728516 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.989124298095703 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.988983631134033 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.988938570022583 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.989008665084839 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.9890785217285156 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.9894680976867676 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.98892879486084 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.989581346511841 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.9892544746398926 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.9894206523895264 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.988901138305664 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.9890878200531006 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.9896774291992188 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.9888594150543213 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.989075183868408 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.98915433883667 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.9888830184936523 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.9889955520629883 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.98901629447937 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.9893953800201416 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.9888975620269775 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.989729642868042 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.9893274307250977 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.9893341064453125 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.9893274307250977 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.989074945449829 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.989069700241089 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.988879442214966 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.989363670349121 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.989429235458374 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.9894800186157227 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.9893863201141357 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.988861083984375 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.988797664642334 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.9886624813079834 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.989293098449707 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.989344835281372 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.9892749786376953 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.9892332553863525 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.9891910552978516 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.9891397953033447 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.988936424255371 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.9892332553863525 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.9892208576202393 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.9889087677001953 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.989064931869507 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.9886155128479004 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.989206075668335 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.988797664642334 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.9893131256103516 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.9894802570343018 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.989400625228882 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.989354133605957 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.9890501499176025 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.988982677459717 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.9890353679656982 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.98925518989563 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.989328622817993 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.989320755004883 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.989302635192871 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.989107131958008 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.9888741970062256 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.989039421081543 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.9893078804016113 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.989363670349121 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.989313840866089 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.989330291748047 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.9889299869537354 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.9892492294311523 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.9886844158172607 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.98925518989563 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.9892446994781494 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.989201784133911 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.9889707565307617 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.9891231060028076 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.988461971282959 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.9891316890716553 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.9892756938934326 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.9894351959228516 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.989367961883545 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.9893722534179688 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.988734722137451 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.989142894744873 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.988715887069702 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.9892406463623047 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.9892940521240234 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.9893600940704346 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.989360809326172 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.9887373447418213 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.9891562461853027 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.9893147945404053 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.989366292953491 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.9893455505371094 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.9892489910125732 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.988682985305786 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.9891204833984375 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.98909330368042 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.9894819259643555 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.9893782138824463 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.9893758296966553 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.9894521236419678 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.9892423152923584 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.989129066467285 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.9885218143463135 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.9893016815185547 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.9893746376037598 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.989356756210327 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.98934268951416 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.9886624813079834 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.9887986183166504 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.9892826080322266 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.989372968673706 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.98942232131958 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.989434242248535 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.989335060119629 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.989321708679199 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.988795757293701 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.989323139190674 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.9892122745513916 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.989243745803833 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.989248275756836 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.989243745803833 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.9882888793945312 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.9884207248687744 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.988530397415161 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.9893617630004883 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.989356756210327 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.9893007278442383 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.989471197128296 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.988717555999756 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.9889183044433594 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.9884607791900635 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.9893126487731934 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.9893693923950195 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.989370346069336 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.989490032196045 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.9890732765197754 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.9888288974761963 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.988673210144043 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.989349126815796 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.989354133605957 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.989321231842041 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.9893245697021484 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.98811936378479 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.9893627166748047 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.9885494709014893 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.9893686771392822 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.9893672466278076 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.989360809326172 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.989272356033325 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.989089012145996 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.989013671875 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.9893293380737305 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.9893362522125244 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.9893863201141357 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.989257574081421 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.9890921115875244 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.988661527633667 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.989065408706665 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.9893274307250977 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.989316940307617 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.989312171936035 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.9893853664398193 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.9880311489105225 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.9887709617614746 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.9890472888946533 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.9893574714660645 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.989388942718506 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.9894111156463623 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.989398717880249 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.988504648208618 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.988600730895996 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.988816499710083 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.9893383979797363 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.9893414974212646 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.9893970489501953 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.989328384399414 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.9888129234313965 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.988917350769043 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.9889075756073 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.989337205886841 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.9893174171447754 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.9898808002471924 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.9893643856048584 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.9893455505371094 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.988438129425049 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.989210844039917 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.989331007003784 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.989150285720825 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.9894258975982666 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.9890072345733643 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.988896608352661 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.9886176586151123 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.989475727081299 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.9894845485687256 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.989161968231201 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.989293098449707 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.988750457763672 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.988602638244629 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.9887380599975586 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.98919415473938 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.98934268951416 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.9889636039733887 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.989222526550293 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.9883649349212646 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.989312171936035 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.988905191421509 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.989315986633301 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.989269733428955 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.9892523288726807 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.9893949031829834 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.988466739654541 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.9892351627349854 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.9892759323120117 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.989077091217041 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.989283561706543 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.9891998767852783 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.989274024963379 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.9886484146118164 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.9887261390686035 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.9891810417175293 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.989163875579834 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.989224672317505 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.989614248275757 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.9892940521240234 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.987661361694336 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.9894137382507324 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.989142894744873 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.989269495010376 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.9892876148223877 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.989215612411499 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.9893929958343506 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.9886398315429688 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.9891321659088135 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.9897193908691406 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.9893102645874023 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.989388942718506 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.989494800567627 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.9892327785491943 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.98606014251709 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.9885873794555664 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.9890239238739014 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.9897992610931396 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.989866018295288 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.9900646209716797 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.98954701423645 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.9875223636627197 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.9895410537719727 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.9891552925109863 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.989344596862793 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.9893686771392822 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.989469051361084 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.98952317237854 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.9880974292755127 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.989020586013794 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.9886622428894043 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.9893083572387695 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.989287853240967 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.9888699054718018 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.989025831222534 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.9881908893585205 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.9891300201416016 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.9883642196655273 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.989490509033203 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.989255666732788 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.989182710647583 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.9893696308135986 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.989755868911743 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.9892501831054688 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.988036870956421 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.8670127391815186 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.8670167922973633 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.867168664932251 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.8669652938842773 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.8670926094055176 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.867605686187744 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.867151975631714 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.8668947219848633 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.8668062686920166 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.8667726516723633 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.8669021129608154 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.8667140007019043 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.8669979572296143 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.86676287651062 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.866762399673462 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.8667709827423096 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.866797924041748 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.8667521476745605 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.8666634559631348 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.866442918777466 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.8667874336242676 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.867043972015381 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.8668901920318604 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.8672027587890625 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.8670194149017334 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.8665802478790283 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.8666906356811523 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.866523265838623 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.866868734359741 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.8672447204589844 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.867022752761841 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.866962194442749 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.867201805114746 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.8669803142547607 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.8668248653411865 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.866929292678833 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.8671927452087402 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.8667633533477783 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.8669729232788086 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.8667445182800293 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.8669207096099854 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.866957902908325 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.867109537124634 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.867114305496216 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.867183208465576 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.8667140007019043 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.8670449256896973 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.866605281829834 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.8670549392700195 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.8671059608459473 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.8669729232788086 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.8672289848327637 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.8668394088745117 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.8669040203094482 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.867095708847046 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.8664863109588623 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.8671257495880127 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.8671748638153076 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.867020606994629 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.8672966957092285 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.8670146465301514 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.866345167160034 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.8670074939727783 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.8671298027038574 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.8669424057006836 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.8672258853912354 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.8671085834503174 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.8671653270721436 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.8668854236602783 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.8667538166046143 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.8670730590820312 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.866971015930176 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.8669281005859375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.8668081760406494 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.866734266281128 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.8670272827148438 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.8666024208068848 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.866917371749878 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.8670432567596436 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.867030143737793 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.8669376373291016 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.8668699264526367 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.86669921875 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.8670156002044678 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.866981267929077 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.8672358989715576 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.866919994354248 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.8665575981140137 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.866607427597046 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.867175579071045 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.866893768310547 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.8669395446777344 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.866971969604492 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.867070436477661 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.8668861389160156 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.867039442062378 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.867255210876465 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.866832971572876 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.8672008514404297 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.8669052124023438 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.8666751384735107 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.866931438446045 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.866826057434082 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.8671183586120605 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.8671491146087646 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.8671178817749023 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.867034435272217 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.866884231567383 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.866901159286499 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.8668808937072754 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.8669114112854004 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.866931438446045 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.8670709133148193 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.866968870162964 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.8669142723083496 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.8668315410614014 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.8671939373016357 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.8668625354766846 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.867145538330078 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.8670687675476074 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.8665831089019775 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.8670477867126465 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.866914987564087 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.8668746948242188 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.8665854930877686 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.8667032718658447 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.8666491508483887 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.8666250705718994 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.8666863441467285 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.866941452026367 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.867124080657959 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.867203712463379 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.867032051086426 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.8667919635772705 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.866827964782715 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.867048740386963 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.8671011924743652 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.8669114112854004 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.8668222427368164 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.8669943809509277 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.8669159412384033 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.8672728538513184 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.8668999671936035 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.866879940032959 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.866971015930176 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.8670156002044678 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.8667304515838623 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.8665876388549805 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.8667845726013184 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.867292642593384 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.8671278953552246 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.866976022720337 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.866809129714966 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.8668365478515625 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.8670573234558105 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.866831064224243 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.8668503761291504 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.8669519424438477 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.8669090270996094 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.8671658039093018 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.867016553878784 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.8666462898254395 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.8669309616088867 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.8672611713409424 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.8668148517608643 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.8669114112854004 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.8668017387390137 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.866878032684326 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.867220401763916 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.867079734802246 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.866837501525879 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.8669886589050293 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.8669509887695312 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.8668007850646973 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.8667352199554443 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.867130994796753 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.8670597076416016 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.8667569160461426 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.8669676780700684 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.866913318634033 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.8670477867126465 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.8671603202819824 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.866837501525879 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.866730213165283 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.8670408725738525 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.866727113723755 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.866922616958618 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.8668394088745117 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.8669002056121826 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.866852283477783 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.8668315410614014 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.866859197616577 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.8670871257781982 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.866957426071167 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.866899251937866 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.866835594177246 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.866734266281128 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.8673980236053467 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.8668112754821777 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.8670239448547363 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.866957426071167 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.8668909072875977 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.8669309616088867 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.8669300079345703 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.866875171661377 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.866528272628784 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.8668975830078125 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.8669240474700928 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.8669769763946533 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.8667802810668945 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.8665931224823 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.8670718669891357 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.8667564392089844 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.8668699264526367 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.866770029067993 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.866856098175049 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.8670687675476074 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.8668198585510254 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.8670365810394287 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.8670175075531006 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.8670241832733154 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.867119789123535 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.867015838623047 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.867100238800049 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.866541624069214 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.866865634918213 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.8668019771575928 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.8669464588165283 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.8670976161956787 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.8671021461486816 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.867053270339966 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.8668465614318848 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.86690092086792 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.8669018745422363 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.866846799850464 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.8668315410614014 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.8667237758636475 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.8666577339172363 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.867318868637085 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.8671000003814697 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.8665852546691895 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.8668477535247803 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.867053270339966 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.8669049739837646 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.8667731285095215 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.8669071197509766 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.86676287651062 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.866442918777466 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.8671436309814453 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.8670554161071777 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.866931438446045 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.867128849029541 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.8666789531707764 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.86680006980896 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.866840362548828 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.866919755935669 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.8669896125793457 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.8670027256011963 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.8669090270996094 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.8669040203094482 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.867034912109375 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.867018461227417 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.8670291900634766 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.8669095039367676 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.867007255554199 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.8669488430023193 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.8669371604919434 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.8666892051696777 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.866647720336914 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.8667874336242676 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.8669538497924805 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.8667564392089844 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.8667709827423096 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.866901159286499 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.8670008182525635 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.867310047149658 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.8668439388275146 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.866814136505127 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.8666956424713135 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.866597890853882 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.8667941093444824 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.866978883743286 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.8668413162231445 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.86692476272583 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.8668882846832275 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.866881847381592 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.866659641265869 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.866884469985962 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.8670127391815186 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.8669960498809814 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.8668668270111084 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.866884469985962 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.866966724395752 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.8665950298309326 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.86700177192688 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.8667337894439697 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.866954803466797 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.866886854171753 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.8670308589935303 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.866755962371826 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.867039442062378 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.866835594177246 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.866539478302002 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.867145538330078 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.86712384223938 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.8672690391540527 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.8670644760131836 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.8667666912078857 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.867043972015381 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.866879940032959 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.8670523166656494 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.8670551776885986 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.867204427719116 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.8668909072875977 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.8669538497924805 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.867194414138794 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.8666107654571533 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.8670156002044678 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.8669216632843018 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.8668572902679443 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.866706609725952 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.8666112422943115 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.8667783737182617 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.866912364959717 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.866852283477783 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.8669021129608154 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.8669915199279785 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.866647720336914 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.867112636566162 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.8671019077301025 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.8669281005859375 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.866964817047119 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.866971015930176 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.866931200027466 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.866687297821045 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.867095470428467 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.8669960498809814 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.8669075965881348 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.866990566253662 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.866955041885376 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.866854190826416 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.8667290210723877 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.8670589923858643 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.8667638301849365 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.8669049739837646 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.866889238357544 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.8670589923858643 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.866976022720337 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.866912364959717 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.8667821884155273 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.866757869720459 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.8670151233673096 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.866995334625244 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.8669216632843018 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.8668010234832764 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.8669803142547607 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.8670432567596436 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.8664190769195557 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.866987705230713 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.8669779300689697 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.8670010566711426 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.866955518722534 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.866706609725952 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.8669986724853516 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.8669793605804443 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.866959571838379 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.866962194442749 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.867098569869995 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.866931200027466 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.8665478229522705 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.866999864578247 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.8666982650756836 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.8669779300689697 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.8670384883880615 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.866954803466797 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.8669638633728027 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.866706609725952 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.866779327392578 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.8665988445281982 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.8669958114624023 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.866950035095215 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.866917848587036 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.866973876953125 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.867079019546509 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.866995334625244 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.867033004760742 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.866969585418701 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.867032051086426 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.867035388946533 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.8670854568481445 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.8667314052581787 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.8671748638153076 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.8669769763946533 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.867033004760742 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.866990327835083 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.8669662475585938 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.8671324253082275 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.866642951965332 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.866832971572876 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.8667237758636475 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.8670384883880615 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.8670082092285156 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.8668737411499023 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.8669822216033936 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.8667140007019043 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.866879940032959 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.8669726848602295 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.866973876953125 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.8669960498809814 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.8670871257781982 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.8669381141662598 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.8667569160461426 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.866779327392578 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.8666467666625977 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.8669753074645996 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.8670196533203125 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.8669657707214355 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.8670027256011963 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.8666625022888184 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.866672992706299 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.866673707962036 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.866952896118164 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.866912364959717 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.8669450283050537 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.8669612407684326 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.8666653633117676 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.8666603565216064 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.8667988777160645 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.866905927658081 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.8669886589050293 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.8669209480285645 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.8669302463531494 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.8666305541992188 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.866831064224243 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.8667237758636475 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.867032051086426 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.866938591003418 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.8669049739837646 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.866960287094116 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.8667407035827637 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.8669593334198 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.866588592529297 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.8669769763946533 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.866992473602295 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.866990566253662 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.867050886154175 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.867063522338867 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.8667333126068115 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.867020606994629 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.8670220375061035 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.8669559955596924 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.866995334625244 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.86696720123291 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.8666436672210693 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.8666956424713135 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.8668606281280518 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.8669967651367188 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.867050886154175 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.8671352863311768 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.8670666217803955 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.8665802478790283 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.8670554161071777 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.86698842048645 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.8670082092285156 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.866926908493042 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.8672401905059814 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.8670403957366943 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.866525173187256 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.8667783737182617 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.8672873973846436 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.8669989109039307 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.8670506477355957 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.8670527935028076 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.867067337036133 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.866548776626587 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.8670737743377686 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.866884469985962 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.8670146465301514 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.8669841289520264 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.867119789123535 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.8669698238372803 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.8663949966430664 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.8669896125793457 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.866818904876709 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.866894483566284 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.866882562637329 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.8667519092559814 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.8668718338012695 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.8668456077575684 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.866954803466797 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.867034912109375 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.866983652114868 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.8669350147247314 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.866926670074463 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.8672127723693848 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.8667821884155273 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.8672850131988525 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.866870880126953 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.8669188022613525 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.8668906688690186 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.866818428039551 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.8670101165771484 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.8665950298309326 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.8671324253082275 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.8667352199554443 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.866910457611084 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.866947889328003 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.8668580055236816 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.867016553878784 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.8664045333862305 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.866908550262451 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.8667988777160645 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.867063045501709 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.8669440746307373 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.8668596744537354 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.866786003112793 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.8663010597229004 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.86710524559021 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.8669612407684326 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.8669216632843018 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.8669464588165283 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.866957426071167 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.8668980598449707 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.8656322956085205 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.866597890853882 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.8668527603149414 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.867367744445801 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.8673739433288574 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.866809368133545 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.867016553878784 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.8669490814208984 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.8671114444732666 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.867187023162842 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.867084264755249 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.8670272827148438 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.866997003555298 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.8670194149017334 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.8663978576660156 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.8668532371520996 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.86665415763855 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.866929292678833 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.8669989109039307 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.866837501525879 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.8669850826263428 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.865818500518799 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.8666374683380127 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.8667685985565186 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.86702823638916 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.8668882846832275 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.866962194442749 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.866821527481079 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.8665425777435303 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.8673489093780518 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.866105318069458 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.811953067779541 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.812067747116089 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.812023878097534 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.8120362758636475 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.812218427658081 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.812202215194702 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.8120412826538086 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.812018632888794 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.8119757175445557 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.811859369277954 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.8117969036102295 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.811918258666992 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.812103033065796 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.8122310638427734 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.811908721923828 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.812023878097534 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.811892509460449 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.8117971420288086 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.8119747638702393 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.8118393421173096 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.812004566192627 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.811976671218872 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.8120157718658447 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.811953067779541 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.8118245601654053 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.8121347427368164 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.8120522499084473 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.8120148181915283 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.8119101524353027 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.8120365142822266 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.8117425441741943 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.8117568492889404 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.8121540546417236 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.8119711875915527 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.812000036239624 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.811941146850586 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.8121237754821777 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.8120956420898438 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.8118433952331543 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.812163829803467 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.811863899230957 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.8120720386505127 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.812004566192627 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.811872959136963 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.8120338916778564 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.81187105178833 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.8122856616973877 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.812018632888794 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.812044143676758 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.812101125717163 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.8120276927948 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.8121256828308105 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.811832904815674 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.811948537826538 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.812068223953247 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.81197452545166 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.8121230602264404 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.8121182918548584 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.811786651611328 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.8117587566375732 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.8115813732147217 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.811863899230957 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.8121345043182373 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.8119964599609375 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.812034845352173 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.8120927810668945 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.8117568492889404 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.8120133876800537 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.8119142055511475 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.811976432800293 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.8120574951171875 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.8121933937072754 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.8120884895324707 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.8119540214538574 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.8120179176330566 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.812209367752075 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.8120310306549072 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.812004566192627 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.812135696411133 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.8119568824768066 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.81164288520813 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.8121092319488525 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.8117592334747314 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.812107563018799 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.811912775039673 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.8119378089904785 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.8122520446777344 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.811957359313965 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.8120205402374268 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.8119077682495117 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.8119888305664062 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.8120040893554688 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.811908006668091 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.8119266033172607 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.811918258666992 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.8117384910583496 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.8119170665740967 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.8119559288024902 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.8120691776275635 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.8119335174560547 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.8120856285095215 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.8118064403533936 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.812035322189331 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.811746597290039 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.812074899673462 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.8120667934417725 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.811816692352295 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.8121328353881836 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.8119630813598633 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.812140703201294 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.812039852142334 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.8119747638702393 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.8119990825653076 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.811976671218872 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.8118319511413574 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.8120064735412598 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.8120226860046387 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.8120529651641846 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.8120484352111816 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.8119702339172363 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.8118464946746826 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.8118247985839844 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.811852216720581 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.812107563018799 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.8119611740112305 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.812030076980591 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.811927556991577 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.8121631145477295 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.812000036239624 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.8119306564331055 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.811715841293335 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.8117470741271973 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.812065839767456 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.8120739459991455 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.8121089935302734 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.8120639324188232 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.8119020462036133 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.8118793964385986 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.8118574619293213 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.8118975162506104 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.811980962753296 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.8120970726013184 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.812206506729126 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.8120505809783936 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.8118298053741455 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.81208872795105 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.812002182006836 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.8120548725128174 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.81180739402771 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.8119940757751465 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.812175989151001 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.8119640350341797 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.811868906021118 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.8119118213653564 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.811990261077881 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.811938762664795 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.812065839767456 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.812082529067993 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.811889410018921 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.811990261077881 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.8120357990264893 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.81199312210083 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.8120317459106445 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.8119068145751953 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.8120317459106445 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.812030076980591 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.8119702339172363 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.812089681625366 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.8118093013763428 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.81182861328125 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.811796188354492 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.8119757175445557 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.811932325363159 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.8117921352386475 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.811983823776245 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.8119313716888428 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.8120555877685547 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.812037229537964 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.8120903968811035 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.8121001720428467 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.8120710849761963 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.81196665763855 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.8118913173675537 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.8118176460266113 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.8118481636047363 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.8119068145751953 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.8119754791259766 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.811875104904175 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.8119099140167236 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.8119049072265625 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.811943769454956 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.812131881713867 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.811983823776245 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.811950922012329 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.8120601177215576 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.8120784759521484 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.8121531009674072 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.8119397163391113 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.8120980262756348 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.8120033740997314 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.8119466304779053 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.811765193939209 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.8120179176330566 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.8122012615203857 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.812054395675659 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.81227707862854 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.8120975494384766 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.8120310306549072 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.812103509902954 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.8119657039642334 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.811952590942383 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.8120064735412598 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.811948537826538 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.8120758533477783 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.8120129108428955 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.811995267868042 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.8120758533477783 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.8117847442626953 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.8120229244232178 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.812167167663574 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.811932325363159 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.8119776248931885 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.811983823776245 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.8118412494659424 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.811840295791626 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.8121092319488525 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.812023639678955 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.81183123588562 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.812039375305176 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.8116557598114014 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.8119983673095703 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.811959981918335 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.8119776248931885 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.811962366104126 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.811899423599243 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.8119406700134277 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.8119540214538574 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.812009334564209 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.812039852142334 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.811943769454956 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.8120474815368652 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.8118276596069336 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.812000036239624 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.8119940757751465 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.8120062351226807 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.8118784427642822 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.811964988708496 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.81189227104187 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.811835765838623 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.8121190071105957 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.8117685317993164 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.812039852142334 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.8120548725128174 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.8120148181915283 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.811972141265869 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.8119289875030518 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.8120267391204834 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.8118317127227783 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.8120033740997314 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.812138557434082 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.8119492530822754 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.8119912147521973 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.811814069747925 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.8119149208068848 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.81201171875 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.811953067779541 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.8119397163391113 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.8121907711029053 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.811936616897583 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.8120219707489014 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.812148332595825 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.8118813037872314 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.811962127685547 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.811882495880127 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.8119313716888428 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.8119425773620605 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.811889410018921 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.8118956089019775 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.8119611740112305 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.8118584156036377 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.8121025562286377 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.8119466304779053 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.8119404315948486 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.8118584156036377 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.812068223953247 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.8119418621063232 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.812082052230835 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.811946153640747 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.8119993209838867 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.8120903968811035 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.8117775917053223 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.811931610107422 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.8118813037872314 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.8119139671325684 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.8119120597839355 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.8118667602539062 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.81195330619812 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.8119561672210693 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.8121285438537598 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.8120357990264893 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.8119571208953857 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.8119053840637207 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.8118855953216553 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.811826705932617 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.812061071395874 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.8120176792144775 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.8120710849761963 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.8120837211608887 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.8119447231292725 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.8118317127227783 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.811842203140259 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.8120055198669434 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.812004327774048 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.812018632888794 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.8119490146636963 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.8117969036102295 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.811997413635254 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.8119304180145264 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.8119819164276123 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.8120739459991455 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.811976909637451 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.8118603229522705 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.8119211196899414 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.8119776248931885 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.8116812705993652 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.8119912147521973 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.811985731124878 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.812183380126953 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.8119568824768066 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.8118700981140137 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.811918020248413 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.8119499683380127 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.812065839767456 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.812034845352173 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.8119711875915527 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.8119611740112305 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.8119964599609375 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.812067747116089 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.8120648860931396 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.8119990825653076 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.811889171600342 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.8121747970581055 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.811988592147827 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.811986207962036 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.8120148181915283 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.81192946434021 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.8120229244232178 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.811805009841919 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.8120052814483643 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.8119826316833496 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.8119993209838867 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.812070846557617 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.811978578567505 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.812049388885498 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.811901569366455 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.811918258666992 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.8117897510528564 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.8120205402374268 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.812091588973999 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.8120803833007812 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.8121068477630615 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.811980962753296 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.8119633197784424 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.8120319843292236 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.8119921684265137 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.81209135055542 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.811983585357666 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.8120195865631104 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.8119266033172607 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.8121531009674072 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.811922550201416 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.8120574951171875 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.8119728565216064 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.812039375305176 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.812119245529175 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.8119256496429443 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.81182861328125 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.811680793762207 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.812034845352173 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.8119161128997803 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.811932325363159 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.8118362426757812 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.8118932247161865 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.811666488647461 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.8120129108428955 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.812072992324829 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.812044620513916 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.8120760917663574 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.811943769454956 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.8121719360351562 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.811946153640747 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.8119940757751465 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.8119921684265137 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.8119871616363525 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.8119583129882812 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.8118112087249756 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.8120005130767822 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.81203293800354 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.8119540214538574 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.8121912479400635 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.812061071395874 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.8117458820343018 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.8120298385620117 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.812049388885498 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.812062978744507 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.812103033065796 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.8119003772735596 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.8119139671325684 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.8119444847106934 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.8120412826538086 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.8119983673095703 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.8120594024658203 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.8120205402374268 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.8119711875915527 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.8119795322418213 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.811739444732666 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.8120994567871094 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.812002182006836 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.8120267391204834 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.8120384216308594 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.8120055198669434 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.8116984367370605 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.8117785453796387 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.812000274658203 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.812039375305176 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.8120741844177246 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.812047243118286 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.8117620944976807 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.8120012283325195 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.8117940425872803 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.812026023864746 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.8120760917663574 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.8119256496429443 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.8119728565216064 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.81164288520813 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.8120381832122803 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.811826705932617 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.8120267391204834 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.8120229244232178 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.8120474815368652 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.812044143676758 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.8120317459106445 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.8118598461151123 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.8121604919433594 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.812023878097534 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.8120362758636475 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.811946153640747 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.8119847774505615 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.8117077350616455 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.811798334121704 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.8119797706604004 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.8120431900024414 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.812101125717163 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.812138557434082 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.8120710849761963 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.8118412494659424 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.812044620513916 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.811950206756592 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.8120052814483643 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.812025785446167 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.8119404315948486 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.8119845390319824 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.8118460178375244 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.811918020248413 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.8119702339172363 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.812093734741211 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.8121750354766846 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.8119852542877197 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.8120765686035156 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.812150001525879 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.811652183532715 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.8120594024658203 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.811958074569702 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.811985731124878 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.811957359313965 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.8120975494384766 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.811612129211426 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.8118059635162354 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.812114953994751 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.8119559288024902 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.8119239807128906 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.811992645263672 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.8118975162506104 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.8119876384735107 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.811938762664795 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.811852216720581 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.8120288848876953 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.8121237754821777 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.8120594024658203 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.811544895172119 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.8118984699249268 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.8119912147521973 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.8119819164276123 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.811951160430908 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.8121016025543213 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.8120431900024414 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.811936616897583 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.811927556991577 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.8120102882385254 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.8118855953216553 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.8120152950286865 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.8118720054626465 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.8119418621063232 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.811483144760132 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.81193470954895 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.8120338916778564 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.812011241912842 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.811954975128174 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.8117940425872803 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.81189227104187 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.811387777328491 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.811798095703125 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.8117294311523438 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.8121025562286377 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.8122661113739014 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.812173843383789 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.8120055198669434 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.8114922046661377 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.8120813369750977 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.8118834495544434 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.811973810195923 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.812002420425415 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.812131404876709 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.812042236328125 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.811657667160034 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.8120083808898926 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.811825752258301 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.8120081424713135 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.8119354248046875 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.811699390411377 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.8119394779205322 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.81173038482666 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.8116745948791504 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.812105178833008 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.8120031356811523 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.8120102882385254 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.8119466304779053 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.812030792236328 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.812103033065796 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.812028408050537 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.8117189407348633 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7788569927215576 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7788515090942383 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7789053916931152 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7788496017456055 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.778892755508423 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.778860330581665 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.778761148452759 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.778714418411255 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7788546085357666 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7787883281707764 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7785069942474365 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.778850555419922 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7787721157073975 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.778820037841797 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.778681993484497 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7788171768188477 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.7787508964538574 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.77873158454895 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7789013385772705 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.7787539958953857 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7787132263183594 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.778726577758789 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.778618574142456 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.7784905433654785 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.77874493598938 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7786648273468018 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7786591053009033 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.778801918029785 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.77899432182312 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.778766632080078 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.7788162231445312 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7786829471588135 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7788567543029785 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7786824703216553 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7785558700561523 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.7789838314056396 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.7789037227630615 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.7788302898406982 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7789392471313477 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7790157794952393 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.7785587310791016 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.778400421142578 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7786033153533936 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.778892755508423 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7785966396331787 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7789058685302734 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.778653621673584 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.7786612510681152 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.778815269470215 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.778920888900757 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7787883281707764 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7786970138549805 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.778644561767578 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.778822183609009 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7787559032440186 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7788407802581787 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.7790017127990723 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.778625011444092 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.778693914413452 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.7786378860473633 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.778810977935791 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.77893328666687 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.778764486312866 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7787044048309326 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7785332202911377 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.778604507446289 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.7788009643554688 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7788004875183105 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.7787091732025146 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.778820514678955 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7788045406341553 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7790114879608154 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7785675525665283 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.778764009475708 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7787692546844482 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.778848648071289 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7788569927215576 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7786316871643066 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.778681993484497 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.778670310974121 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.778895854949951 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.778808116912842 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.7788829803466797 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.779042959213257 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.778642177581787 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.778795003890991 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7788515090942383 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7789573669433594 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.7787153720855713 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.778670310974121 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7787399291992188 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7788212299346924 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7789251804351807 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7788610458374023 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7787561416625977 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.778777837753296 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.779005527496338 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7786693572998047 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.778872013092041 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.778799533843994 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7788174152374268 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.778820037841797 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7788522243499756 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7788684368133545 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.77880597114563 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.778775215148926 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.7788515090942383 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.778984785079956 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.778782844543457 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7789249420166016 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.778827428817749 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7785487174987793 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.778865098953247 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.778604507446289 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7786495685577393 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.7788891792297363 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7789080142974854 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.7789089679718018 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.778848886489868 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7787277698516846 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7787609100341797 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.778838634490967 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.778801918029785 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.7786495685577393 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.778679132461548 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7787837982177734 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.7787656784057617 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7788162231445312 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7788679599761963 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7787415981292725 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7787909507751465 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7785935401916504 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.7787487506866455 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.7787399291992188 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.778773069381714 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.7789435386657715 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.778796434402466 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7786169052124023 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7788140773773193 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7787134647369385 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7787649631500244 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7788124084472656 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.778897285461426 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.778655767440796 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.778895378112793 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7787890434265137 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7789053916931152 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.7788140773773193 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.778994560241699 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.7787346839904785 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.7788426876068115 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.778846502304077 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7788333892822266 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7788751125335693 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7787439823150635 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.7789392471313477 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.7788233757019043 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7786521911621094 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.77888560295105 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.778801202774048 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.77874493598938 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.7788028717041016 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7787318229675293 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.778663158416748 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.778749942779541 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7787365913391113 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7787187099456787 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7787766456604004 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.778703451156616 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.7789385318756104 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.778743028640747 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.778750419616699 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.778801202774048 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.7788383960723877 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7788119316101074 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.7785496711730957 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.7787539958953857 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.778663158416748 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.778803586959839 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.778757095336914 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.778687000274658 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.778872489929199 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7787559032440186 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.77878999710083 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.778822422027588 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.778944253921509 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7789130210876465 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.778846502304077 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.778843402862549 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7788619995117188 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7788774967193604 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.7789242267608643 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.778902769088745 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.7786293029785156 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7788262367248535 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.778881072998047 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7788071632385254 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7789111137390137 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.778921127319336 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.778857707977295 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.778954029083252 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.778743028640747 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7788424491882324 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.77868914604187 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.778803586959839 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7786757946014404 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.778670310974121 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.77895450592041 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7789466381073 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7787978649139404 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7788312435150146 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7788164615631104 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.778658628463745 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.778761625289917 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7787559032440186 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7788326740264893 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.778864860534668 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7787816524505615 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.778895854949951 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.778623580932617 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.778773784637451 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7786340713500977 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7788233757019043 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7788071632385254 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.7789371013641357 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.778762102127075 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.778897285461426 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.778902769088745 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.778799057006836 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.7788004875183105 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.778740882873535 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.778691053390503 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.7787866592407227 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.7785513401031494 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.7787692546844482 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7787253856658936 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.778838634490967 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.778728723526001 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.7788872718811035 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7788333892822266 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7785911560058594 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7787747383117676 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7787020206451416 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7787044048309326 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.7787375450134277 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.778801202774048 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.778872013092041 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.778866767883301 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7786717414855957 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7788021564483643 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.7788171768188477 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.778866767883301 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7785446643829346 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.7786717414855957 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.77880597114563 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7787630558013916 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.778782844543457 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.7787487506866455 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.77868914604187 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7787697315216064 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.77873158454895 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7789666652679443 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7788610458374023 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7789313793182373 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.7788476943969727 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.7787256240844727 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.778839588165283 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7785842418670654 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.778740882873535 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.7788476943969727 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.77884578704834 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.778852939605713 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.778815984725952 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.778855800628662 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7787978649139404 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.778799057006836 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7788619995117188 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7788357734680176 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7786879539489746 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.77874493598938 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7786905765533447 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7788150310516357 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7787349224090576 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7788333892822266 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7787253856658936 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.778698205947876 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7787981033325195 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.778737783432007 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7788124084472656 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7787513732910156 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7787153720855713 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.778865098953247 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.778749465942383 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.778756856918335 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7787652015686035 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7787771224975586 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7786941528320312 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.77864408493042 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.778764486312866 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7787530422210693 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7787671089172363 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.7788264751434326 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.778696298599243 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7786431312561035 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7787020206451416 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7788243293762207 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7788522243499756 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.77866268157959 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7788496017456055 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.778414487838745 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.778841495513916 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7788245677948 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7788100242614746 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7788870334625244 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.778730630874634 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7787938117980957 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.778993606567383 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.778714418411255 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7787716388702393 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7787539958953857 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7787134647369385 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.7787253856658936 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7788071632385254 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.7786593437194824 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7787721157073975 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7787609100341797 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.7787747383117676 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7787399291992188 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7786591053009033 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.778780221939087 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.7788643836975098 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.778796434402466 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7788569927215576 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.778759479522705 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.778958559036255 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.7787206172943115 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7785544395446777 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.778843641281128 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7788352966308594 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7788243293762207 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.7786552906036377 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.778730630874634 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.778820276260376 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7788000106811523 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7788150310516357 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.77882719039917 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.778879165649414 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7787227630615234 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7786004543304443 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.77878737449646 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7787792682647705 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7788093090057373 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.778813362121582 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.778630018234253 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.7787258625030518 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.77878475189209 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.778768539428711 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.778787851333618 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7788476943969727 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7786002159118652 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7786953449249268 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.778782844543457 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.778838634490967 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.778749942779541 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7786738872528076 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.778701066970825 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.778590202331543 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.778759479522705 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7787907123565674 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7787928581237793 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.778747797012329 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7788314819335938 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.778787851333618 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.778700113296509 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.778947591781616 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.7787561416625977 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.778771162033081 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.778780937194824 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7788286209106445 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7786848545074463 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7788000106811523 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.778862953186035 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.778757095336914 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7788052558898926 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.778820514678955 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.778625726699829 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.77864408493042 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.778822422027588 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.778761625289917 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7787423133850098 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7787981033325195 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.7785487174987793 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7787225246429443 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.778674840927124 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.778809070587158 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7788100242614746 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.778754949569702 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.778762102127075 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7785167694091797 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.778796434402466 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.7784907817840576 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7787325382232666 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.7788281440734863 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7787909507751465 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7788658142089844 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7787954807281494 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.7787041664123535 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.778773069381714 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7787747383117676 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.778820514678955 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.778797149658203 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.778749942779541 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.778629779815674 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.778740644454956 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.778721809387207 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7787585258483887 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7788050174713135 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.778773784637451 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.778735637664795 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7785487174987793 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.7785468101501465 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7788586616516113 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.778866767883301 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7788617610931396 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7787909507751465 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.7785801887512207 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.778791904449463 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.778590202331543 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7787044048309326 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7787296772003174 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7787041664123535 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7784793376922607 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7786097526550293 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.778527021408081 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7787468433380127 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.778764009475708 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.7787554264068604 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.778848886489868 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.7788143157958984 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.7787487506866455 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.778724431991577 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7786784172058105 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.778712511062622 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7787325382232666 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7786707878112793 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7784423828125 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.7786951065063477 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.7787458896636963 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.7787091732025146 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7788305282592773 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.7788233757019043 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.7786824703216553 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.778799533843994 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7786810398101807 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.778766632080078 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.7786829471588135 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.77878999710083 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.7787423133850098 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.778615951538086 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7787721157073975 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7787091732025146 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7787671089172363 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.778799533843994 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7788405418395996 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7788679599761963 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7784459590911865 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7787163257598877 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.778963565826416 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.778806209564209 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7787439823150635 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7787346839904785 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.778825283050537 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.778806209564209 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.778815984725952 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.778740882873535 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.7788326740264893 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.7787437438964844 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.7788171768188477 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.778773307800293 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.778698205947876 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7786779403686523 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7786593437194824 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7787206172943115 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7787134647369385 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.778733730316162 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.77873158454895 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.778473138809204 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.778780221939087 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7786571979522705 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7786991596221924 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7787489891052246 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.778804063796997 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.778677463531494 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7788352966308594 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.7787277698516846 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.7790026664733887 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7587106227874756 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7588627338409424 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.758840560913086 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.758925199508667 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.75884747505188 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.758901834487915 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.758840560913086 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7588655948638916 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.758852005004883 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.7588412761688232 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7588279247283936 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.7588601112365723 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.758765935897827 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7588984966278076 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.7588279247283936 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7587950229644775 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.7587051391601562 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7589309215545654 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.758718967437744 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.7587435245513916 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.75888729095459 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7587695121765137 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.7589104175567627 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.7588951587677 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.7586288452148438 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7587363719940186 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.7588679790496826 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.7587642669677734 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.758671283721924 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7587857246398926 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7587740421295166 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.758713960647583 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7588202953338623 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.758861780166626 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.758666515350342 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.7588205337524414 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.7586934566497803 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7588746547698975 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7588584423065186 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.758850336074829 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7589075565338135 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7588160037994385 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.758845806121826 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7588932514190674 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7588350772857666 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7588672637939453 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.758913516998291 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7588162422180176 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.75869083404541 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.758758544921875 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.758735179901123 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7587404251098633 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7588484287261963 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.758755683898926 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.758591651916504 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7586774826049805 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.758845806121826 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.758667469024658 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.758753538131714 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.758822441101074 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7587597370147705 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.758904457092285 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7586240768432617 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.758981227874756 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.758784770965576 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.758882761001587 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.7588119506835938 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.7587239742279053 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7586398124694824 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.758680582046509 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.758802890777588 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.758790969848633 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.7588515281677246 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7588233947753906 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7586686611175537 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.758671760559082 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.758824586868286 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7588624954223633 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.7587037086486816 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.7586286067962646 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.7585296630859375 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7588608264923096 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7587695121765137 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.758850336074829 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7587668895721436 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.7588751316070557 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7586727142333984 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7588181495666504 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.758835792541504 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7587032318115234 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.7588279247283936 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.758730173110962 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7587838172912598 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.7587618827819824 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7587859630584717 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7587363719940186 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.758742094039917 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7586960792541504 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.758917808532715 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.758779525756836 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.758742570877075 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.758726119995117 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7587575912475586 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7586781978607178 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7588555812835693 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7588601112365723 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7588484287261963 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.7586662769317627 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.758897304534912 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.758810043334961 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.75874662399292 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.758774518966675 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7588000297546387 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7587006092071533 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.7585248947143555 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.758824586868286 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.7587382793426514 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.758826971054077 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7587146759033203 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7589049339294434 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.758758783340454 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7587990760803223 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7587926387786865 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.758850574493408 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.7587695121765137 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.758786678314209 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.758819341659546 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.7587592601776123 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7588980197906494 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7589752674102783 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.758784532546997 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7586934566497803 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.758728265762329 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.75876522064209 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.758845806121826 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.7588438987731934 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7587835788726807 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7587058544158936 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.758793830871582 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.7587249279022217 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.7586405277252197 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.758779764175415 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.758821487426758 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.758723020553589 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.758694887161255 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7587404251098633 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.758582353591919 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7588112354278564 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.758795976638794 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7587344646453857 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.758789300918579 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.758723020553589 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7587006092071533 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7587077617645264 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.758680582046509 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.75876522064209 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7588555812835693 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7587344646453857 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.758831024169922 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.758746385574341 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.758681535720825 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.7587435245513916 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.7587735652923584 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.758852005004883 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7588083744049072 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.758821487426758 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.758685350418091 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7588438987731934 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.7588019371032715 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.75879168510437 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7588000297546387 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.758948802947998 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.758718967437744 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.758888006210327 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.7588062286376953 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.758786916732788 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7586684226989746 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.758707284927368 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7587273120880127 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.758739471435547 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.7586028575897217 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7587928771972656 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7586605548858643 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.758843183517456 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.758587598800659 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.758885145187378 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7589690685272217 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.758805274963379 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.758723258972168 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.758802890777588 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.758687973022461 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.75856351852417 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7588233947753906 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7587015628814697 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.758676767349243 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.758779525756836 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.758648157119751 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7588489055633545 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.7587292194366455 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.75879168510437 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7586541175842285 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7587051391601562 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7586898803710938 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.758800983428955 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.758786916732788 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7588109970092773 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.758828639984131 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7587978839874268 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7588729858398438 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.758697986602783 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7588284015655518 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.7587523460388184 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.758871078491211 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.7587223052978516 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.7588019371032715 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.758692502975464 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.758854627609253 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7588284015655518 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.75874924659729 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.758782148361206 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.758847951889038 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7588284015655518 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.758819103240967 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.758789300918579 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7585785388946533 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.758730173110962 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.758822441101074 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.758664608001709 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.758873701095581 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.758901357650757 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.758765935897827 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.758687734603882 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.758845329284668 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.758924722671509 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.7586991786956787 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7587058544158936 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7588913440704346 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7587344646453857 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7588143348693848 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.75877046585083 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.75883150100708 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7586581707000732 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.758695125579834 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.758635997772217 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7586898803710938 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.758820056915283 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7588295936584473 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.758695602416992 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7587497234344482 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.758924722671509 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7586863040924072 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7587616443634033 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7587990760803223 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.7587785720825195 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.758850336074829 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7586863040924072 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.7587525844573975 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7587969303131104 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7588438987731934 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7588000297546387 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.758680820465088 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.758715867996216 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.7586605548858643 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7589662075042725 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7588109970092773 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.758728265762329 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.7588679790496826 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.758742094039917 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.75882625579834 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.758667469024658 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7587473392486572 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7588324546813965 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.7587413787841797 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7587990760803223 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.758761167526245 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.758784770965576 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.75872802734375 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7587571144104004 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7588253021240234 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7587625980377197 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.758817434310913 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.758833408355713 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7586705684661865 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7587320804595947 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7588565349578857 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7588751316070557 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7588372230529785 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7587740421295166 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.758695125579834 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7587339878082275 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7587554454803467 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7587709426879883 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7587127685546875 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.75882625579834 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.758580207824707 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7588512897491455 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7586677074432373 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.758765697479248 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7588376998901367 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.7586405277252197 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7586827278137207 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7587554454803467 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7588319778442383 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7588162422180176 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7587106227874756 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.7588648796081543 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7587273120880127 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.7590277194976807 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7587673664093018 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7587811946868896 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7587485313415527 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7587733268737793 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.758805274963379 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.758598804473877 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.758692502975464 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7586469650268555 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7588090896606445 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.758866548538208 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7586798667907715 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.758822202682495 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7587897777557373 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7587740421295166 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7587811946868896 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.758793830871582 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.758777141571045 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.758711338043213 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7586827278137207 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.758742094039917 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.75881028175354 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.758746385574341 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.758786678314209 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.758516788482666 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.75864315032959 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7587804794311523 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.7587852478027344 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.758756160736084 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7587249279022217 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7587530612945557 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.758800983428955 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7587451934814453 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.758817195892334 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7588324546813965 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7588679790496826 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7587764263153076 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.75874924659729 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7588119506835938 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.7587530612945557 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.75876784324646 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.758708953857422 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7587435245513916 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.758777618408203 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.758652448654175 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.7587714195251465 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.758822441101074 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.758753538131714 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7587804794311523 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.75876522064209 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.758790969848633 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7587785720825195 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.75877046585083 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7588493824005127 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7587714195251465 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.7586169242858887 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.7587642669677734 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.758525848388672 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7587482929229736 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.758788824081421 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.758859395980835 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7586209774017334 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.758772373199463 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.758589744567871 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.758807897567749 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7587969303131104 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7587997913360596 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.758774995803833 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7587947845458984 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7588376998901367 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7586119174957275 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.7588143348693848 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.7587828636169434 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.758819341659546 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.7585713863372803 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7587966918945312 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.7587413787841797 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.758812427520752 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7587714195251465 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.758781671524048 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.758732318878174 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7587547302246094 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.7588133811950684 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.758838176727295 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7587473392486572 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.758774995803833 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.758695125579834 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.7589356899261475 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7588162422180176 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.758782148361206 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7587971687316895 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.758840560913086 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.758479118347168 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.758807897567749 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.758709669113159 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7587673664093018 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.758777141571045 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.758819103240967 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.7588119506835938 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.7586894035339355 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.7588043212890625 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7586827278137207 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.75877046585083 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.758817434310913 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.758784532546997 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7588448524475098 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7586565017700195 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.758920192718506 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.758819103240967 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.758852958679199 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.758842706680298 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7587811946868896 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7588627338409424 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.758753538131714 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.7585761547088623 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.758662223815918 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.758763313293457 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7587313652038574 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7588698863983154 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7586944103240967 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7588071823120117 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.7588467597961426 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7587764263153076 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.758737325668335 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7587571144104004 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.758532762527466 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7587454319000244 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.7588746547698975 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7587320804595947 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7587900161743164 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.7588601112365723 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7587578296661377 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7587544918060303 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7589023113250732 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.758735418319702 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.758798122406006 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.75874924659729 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.758800983428955 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.758641242980957 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.75876784324646 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.758819341659546 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.758824348449707 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.758726119995117 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.758843183517456 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7585527896881104 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7588584423065186 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7587966918945312 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7587785720825195 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7587759494781494 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.758760690689087 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.758725881576538 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.75856614112854 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7586429119110107 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.758655071258545 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.7588882446289062 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.758915662765503 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.758843183517456 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.7587687969207764 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.758679151535034 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7588231563568115 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.758749485015869 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.758755683898926 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.758852958679199 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.7588367462158203 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.758790969848633 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.7586307525634766 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7587432861328125 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.758852005004883 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7587497234344482 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.758779525756836 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.7587339878082275 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.758721113204956 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.7585947513580322 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.758655071258545 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7589023113250732 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7588584423065186 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.758791923522949 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7587263584136963 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7587525844573975 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7585508823394775 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.758737325668335 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.758657455444336 + } + ] + }, + { + "bit": 9, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7460079193115234 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7459137439727783 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7460012435913086 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.745919942855835 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.7460267543792725 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.745964527130127 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7460567951202393 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.745893716812134 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.745973587036133 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7458226680755615 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7458834648132324 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.745954990386963 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7457873821258545 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.74584698677063 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.745929718017578 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.745967388153076 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.7458319664001465 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.745859146118164 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.7458584308624268 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7457480430603027 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7459187507629395 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.745943069458008 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.745853900909424 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.745858669281006 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.745988368988037 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.7458038330078125 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.745931625366211 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.746058225631714 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7459752559661865 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7458484172821045 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.7457404136657715 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.7459986209869385 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.745879650115967 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.7460591793060303 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.745997667312622 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.7459359169006348 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.745944023132324 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7458314895629883 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7459731101989746 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7458388805389404 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.745884418487549 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.746018648147583 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.746027708053589 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.7459211349487305 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.745856523513794 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7459208965301514 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.745854377746582 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7458512783050537 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7459659576416016 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.7460074424743652 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7459046840667725 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.745788335800171 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.746006488800049 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.74587345123291 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.745837926864624 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.745811700820923 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7458858489990234 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7458858489990234 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.7458858489990234 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7458128929138184 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7459065914154053 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.7459099292755127 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7458293437957764 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7458994388580322 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.745856523513794 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7460832595825195 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.7460501194000244 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7459843158721924 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.74582576751709 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.745774507522583 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.74592661857605 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.745922803878784 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.746027946472168 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.7459118366241455 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.746087074279785 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7458600997924805 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.745919704437256 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.7458906173706055 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7459099292755127 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.745893955230713 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.7459940910339355 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7459664344787598 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7457985877990723 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.745819568634033 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.74588680267334 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7459325790405273 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.7459731101989746 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.745927572250366 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.745912790298462 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.745840072631836 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.745833158493042 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.745924949645996 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.745912790298462 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.745889186859131 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7459189891815186 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.7459030151367188 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.746087074279785 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.745816230773926 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.745859146118164 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.745873212814331 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7460176944732666 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.7458081245422363 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7456939220428467 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.745994806289673 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7460386753082275 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.745924472808838 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.745976448059082 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.745795726776123 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.745847463607788 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.745999813079834 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.745983362197876 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7458958625793457 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7459557056427 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.7459189891815186 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.745884418487549 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7458553314208984 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.745924949645996 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.745924472808838 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.745901584625244 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.7458512783050537 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.745912551879883 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.7459030151367188 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.745919704437256 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7459771633148193 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7460103034973145 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.74592924118042 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7458078861236572 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.745903730392456 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7458040714263916 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7459511756896973 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.745765447616577 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.745739459991455 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.745964527130127 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7459757328033447 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.745943784713745 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.745999574661255 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.745896816253662 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7457664012908936 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7460074424743652 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.745967388153076 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.7458784580230713 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.7459261417388916 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7459397315979004 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.7457878589630127 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7459261417388916 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7459158897399902 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.7458627223968506 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7458913326263428 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.745830774307251 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7459397315979004 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7457921504974365 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.745832920074463 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.745877504348755 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.74584698677063 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.7459006309509277 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.7459218502044678 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.7459700107574463 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.745929002761841 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7458627223968506 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7457802295684814 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7458274364471436 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7459168434143066 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7459468841552734 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.746037006378174 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.7460381984710693 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.745756149291992 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.745861053466797 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7460057735443115 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.745877742767334 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7459166049957275 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7459940910339355 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.745877265930176 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.7459797859191895 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.7460126876831055 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.745964527130127 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.745927572250366 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7458856105804443 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.746032476425171 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7460269927978516 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.7459583282470703 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7458713054656982 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.745896816253662 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7459189891815186 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7458081245422363 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.7458701133728027 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.7458298206329346 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7459752559661865 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7458388805389404 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.74601674079895 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7458622455596924 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.745887517929077 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.745781660079956 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.745835542678833 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.745741605758667 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7458319664001465 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.745898485183716 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.7459044456481934 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7459518909454346 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.745953321456909 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.745893716812134 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.7458701133728027 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.7459208965301514 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.74595308303833 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.745934247970581 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.7458701133728027 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.74585223197937 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7458884716033936 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.745859146118164 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.7459523677825928 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.745758533477783 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.7459611892700195 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.7459075450897217 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7458746433258057 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.7459189891815186 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.745927572250366 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.745978355407715 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.745919942855835 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.7458693981170654 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.7459378242492676 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.746004819869995 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7459373474121094 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7457988262176514 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.745933771133423 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7459490299224854 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.7459347248077393 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.745856285095215 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.7459914684295654 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.7460410594940186 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.7457571029663086 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7459583282470703 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.745938777923584 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7459521293640137 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7459709644317627 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7459716796875 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.7460269927978516 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.7459020614624023 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.745941400527954 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7458584308624268 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.745889663696289 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.745882749557495 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7459044456481934 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7457873821258545 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7458584308624268 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7459206581115723 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.745879650115967 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7458715438842773 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7459895610809326 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.745893955230713 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7459805011749268 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.745800733566284 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7456743717193604 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7458953857421875 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7460248470306396 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.745922088623047 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7458200454711914 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.7460217475891113 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.746035099029541 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.745817184448242 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7457501888275146 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.745901346206665 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7460103034973145 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7459399700164795 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.7459306716918945 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.7457833290100098 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.745790481567383 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.745866298675537 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.7458958625793457 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.7459206581115723 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.74596905708313 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.745957374572754 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7460012435913086 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.745912790298462 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.7460339069366455 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.7458298206329346 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.7460296154022217 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.7457454204559326 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.7458529472351074 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7458393573760986 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.745905637741089 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7459166049957275 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7458763122558594 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.74591064453125 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.745944023132324 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7457289695739746 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.745877265930176 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.745965003967285 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7458200454711914 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.745990514755249 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.745882511138916 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7459211349487305 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7459957599639893 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.7459709644317627 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7459301948547363 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7459864616394043 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7459068298339844 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.7459218502044678 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7459561824798584 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.745966911315918 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.745755672454834 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.745894432067871 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7458460330963135 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7458441257476807 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7459499835968018 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.745893716812134 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.745961904525757 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.7458090782165527 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7460057735443115 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.7457985877990723 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.745948314666748 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7459967136383057 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7459967136383057 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.745988607406616 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.745957374572754 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.7458691596984863 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7458252906799316 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.74594783782959 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.7459206581115723 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.7459664344787598 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.7459158897399902 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7458279132843018 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.7460246086120605 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7458362579345703 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7458853721618652 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.745914936065674 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7460012435913086 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.745913028717041 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.745778799057007 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.7457797527313232 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7457921504974365 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.745889663696289 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7458994388580322 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.745887517929077 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7459166049957275 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7457687854766846 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7459261417388916 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7457189559936523 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7459306716918945 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.745929002761841 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.745922803878784 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7458314895629883 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.745797634124756 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7457823753356934 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7458574771881104 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.745985269546509 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7459237575531006 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7458765506744385 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7459471225738525 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7460410594940186 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7457234859466553 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7459137439727783 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7459280490875244 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.745922803878784 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7459139823913574 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7458853721618652 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.7458927631378174 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7458794116973877 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.745795488357544 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7459347248077393 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.745967388153076 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.745931625366211 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.745880365371704 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7459280490875244 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.7455902099609375 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7458372116088867 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7459306716918945 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.745943784713745 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7459092140197754 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.7459092140197754 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7457571029663086 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.745896816253662 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7460062503814697 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7459170818328857 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7459280490875244 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7459092140197754 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.7459006309509277 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.745905637741089 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.7458324432373047 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.745823383331299 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7459521293640137 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7459490299224854 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7459099292755127 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.7459635734558105 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7459628582000732 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.746011972427368 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.7459218502044678 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.7458953857421875 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.7459301948547363 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7460217475891113 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.745954990386963 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.7459805011749268 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.746026039123535 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7457962036132812 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.745953321456909 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.745957374572754 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.745948076248169 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.745965003967285 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.7458574771881104 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7458391189575195 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.7458057403564453 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7459306716918945 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7459189891815186 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.745927095413208 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.746002197265625 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.74584698677063 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7459449768066406 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.745728015899658 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.7458388805389404 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7459492683410645 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7459957599639893 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.745845079421997 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.7457823753356934 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.7459449768066406 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.745884418487549 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.745936870574951 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.745896816253662 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.745910882949829 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7459099292755127 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7457211017608643 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.74594783782959 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.745948314666748 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.745979070663452 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.7459681034088135 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7459280490875244 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7459726333618164 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.745830774307251 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.745861053466797 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.7458109855651855 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7459628582000732 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.745950222015381 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.745992422103882 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7459709644317627 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.746096611022949 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.7459917068481445 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.7459237575531006 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.7459940910339355 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.745918035507202 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.745938777923584 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7458927631378174 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.745892286300659 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7459449768066406 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7459986209869385 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.745906114578247 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.745914936065674 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.74594783782959 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.7459046840667725 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.745828866958618 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7458555698394775 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.745879650115967 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.745938777923584 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.745988368988037 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7459540367126465 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.7459986209869385 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.745765447616577 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7458605766296387 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.7458419799804688 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.745906114578247 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7459030151367188 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.745954990386963 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.7459557056427 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.745818614959717 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.745966911315918 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.745802879333496 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7459378242492676 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.74596905708313 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7459397315979004 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.7459676265716553 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.745967388153076 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.74588680267334 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.7458505630493164 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7459421157836914 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7459957599639893 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.746013641357422 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.7459933757781982 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.7458815574645996 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.7458314895629883 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7459092140197754 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.74594783782959 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7459118366241455 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.74595308303833 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7460415363311768 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.745854616165161 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7457385063171387 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.745960235595703 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.745994806289673 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.7459933757781982 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7460379600524902 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7459654808044434 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.745847463607788 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.7461931705474854 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.7457854747772217 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.745936393737793 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.745954990386963 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.745912790298462 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.745842218399048 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.745896816253662 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7458691596984863 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.745943069458008 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7459325790405273 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7459592819213867 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.745870351791382 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7460038661956787 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.7460334300994873 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7459726333618164 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.745901346206665 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.745961904525757 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.745944023132324 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.7459003925323486 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.745907783508301 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.745882749557495 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.745717763900757 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.745868444442749 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.745896816253662 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.7459428310394287 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.745936870574951 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.7460334300994873 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.7458202838897705 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.745936870574951 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7459397315979004 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.7458858489990234 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.745981454849243 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.7459325790405273 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.745880603790283 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7457480430603027 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7459325790405273 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7460436820983887 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7459876537323 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.745938777923584 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.746006965637207 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7459731101989746 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7458949089050293 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.746023654937744 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.7461023330688477 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.745910882949829 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7459564208984375 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.745934247970581 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.746013641357422 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.745836019515991 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7460289001464844 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7458877563476562 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.7460360527038574 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.7459635734558105 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.745931625366211 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.7459564208984375 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.7458016872406006 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7459421157836914 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.745778799057007 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7459325790405273 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7458529472351074 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.745840072631836 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.7458691596984863 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.745774507522583 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.7460198402404785 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7458388805389404 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.745936393737793 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.745948314666748 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7458584308624268 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7459218502044678 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7460851669311523 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.745908737182617 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.746156692504883 + } + ] + }, + { + "bit": 10, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.734651803970337 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7346246242523193 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7345993518829346 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7347168922424316 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.7347025871276855 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.734712600708008 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.734713077545166 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.734503984451294 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7346384525299072 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7346696853637695 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7345573902130127 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.734508514404297 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7346718311309814 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.7346813678741455 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7346081733703613 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.734687566757202 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.734567165374756 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7346575260162354 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.734628677368164 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.734788179397583 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7345879077911377 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.7346315383911133 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.734628677368164 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7346110343933105 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.7346184253692627 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.734637498855591 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.7346174716949463 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.734455108642578 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7345850467681885 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7346110343933105 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.7346115112304688 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.7346084117889404 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.7346677780151367 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.734422206878662 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.734473705291748 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.734623432159424 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.734545946121216 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7346465587615967 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7346224784851074 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.734651803970337 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7345943450927734 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.7346038818359375 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.7346363067626953 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.734652519226074 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.734692335128784 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.734619617462158 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.7345316410064697 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.734729766845703 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7345869541168213 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.734668493270874 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7346866130828857 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.734746217727661 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7347068786621094 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.734626054763794 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.734614372253418 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.734466552734375 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7345919609069824 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7346878051757812 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.734693765640259 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7345762252807617 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7345921993255615 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.734541893005371 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.734743595123291 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.73460054397583 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7346115112304688 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.734637498855591 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.7346112728118896 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.734466552734375 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.7347025871276855 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.7346205711364746 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.734607696533203 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.7346205711364746 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.734586000442505 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.7346205711364746 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7345261573791504 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7345638275146484 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.734766721725464 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.734614133834839 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.734631061553955 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.734578847885132 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.7346131801605225 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7345783710479736 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.734757900238037 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7346384525299072 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.7345941066741943 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7345306873321533 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.734598398208618 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7345693111419678 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7345988750457764 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.7346436977386475 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7346224784851074 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.734626054763794 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7346606254577637 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.7345502376556396 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7345476150512695 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.734626054763794 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.7343239784240723 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7347779273986816 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.73465633392334 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7346062660217285 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7345454692840576 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.734571695327759 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7346277236938477 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.73445725440979 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.734511613845825 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.734619617462158 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7346749305725098 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7346625328063965 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.734722137451172 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.7345542907714844 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.734618902206421 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7347631454467773 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7347075939178467 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.734562873840332 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7345168590545654 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7346465587615967 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7346293926239014 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7345476150512695 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.734731912612915 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.734624147415161 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.7345774173736572 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.7345118522644043 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.7345821857452393 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7345528602600098 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.734572649002075 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.734726905822754 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7345731258392334 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.734689950942993 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.734631299972534 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7345244884490967 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.734598398208618 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7347068786621094 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.734593152999878 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7345943450927734 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.7346696853637695 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.734736204147339 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7347331047058105 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.734691619873047 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.734464645385742 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.734560012817383 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.734571695327759 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.734668731689453 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.734668731689453 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.7346909046173096 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7346837520599365 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7346553802490234 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.7345476150512695 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7345757484436035 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7346346378326416 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7345516681671143 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7346062660217285 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7347190380096436 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.734701156616211 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.734699010848999 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.7346363067626953 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.734612464904785 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.734658718109131 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.734631061553955 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.734694719314575 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.734778642654419 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.73468279838562 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7345762252807617 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7346341609954834 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.7346460819244385 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.734605550765991 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.734747886657715 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.7346549034118652 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7345590591430664 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7346019744873047 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.734591245651245 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7345821857452393 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.734595775604248 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.734534978866577 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.73459529876709 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.734614133834839 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.7344887256622314 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7346153259277344 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.7344677448272705 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.734738826751709 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.734503984451294 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7345993518829346 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7346246242523193 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7345919609069824 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7346818447113037 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.7347211837768555 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.734705686569214 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.73459529876709 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7344372272491455 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.734576940536499 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7346031665802 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7345964908599854 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7346229553222656 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.734621047973633 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.7346272468566895 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7345573902130127 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.7346420288085938 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.7345798015594482 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7345380783081055 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7345335483551025 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.734689474105835 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.734659433364868 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.7345290184020996 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7345619201660156 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.734619617462158 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.734606981277466 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.734617233276367 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7346386909484863 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.7347497940063477 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.734598398208618 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.7347118854522705 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.7346103191375732 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.7345893383026123 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7346155643463135 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.734612464904785 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7345056533813477 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.7345821857452393 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7345428466796875 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.734565258026123 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.734623670578003 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.7346887588500977 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7346534729003906 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7346677780151367 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.7344822883605957 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.734578847885132 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.7346556186676025 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7347004413604736 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.734626054763794 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.7345449924468994 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.734631299972534 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.734619379043579 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.734464645385742 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7347569465637207 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7345831394195557 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7345528602600098 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.734548568725586 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.7346436977386475 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.734578847885132 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.734508752822876 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.7345521450042725 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.734612464904785 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.734487295150757 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7345988750457764 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7347025871276855 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7346866130828857 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7346062660217285 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7346737384796143 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7345528602600098 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7345669269561768 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7345943450927734 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.7346420288085938 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7346980571746826 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7346482276916504 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.734637498855591 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.7346277236938477 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7346885204315186 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.73449969291687 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.7345244884490967 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.734694719314575 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.734668731689453 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.7345850467681885 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7346394062042236 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.734654426574707 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.7345476150512695 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.734593152999878 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.734618902206421 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7347471714019775 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.734628677368164 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.7345964908599854 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.734598398208618 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.7345709800720215 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.734447479248047 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.7345590591430664 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.734541654586792 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.734635591506958 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.7346322536468506 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.734623670578003 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.7346434593200684 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.734565019607544 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.734748363494873 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7346436977386475 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.734522819519043 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.734494924545288 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7345387935638428 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7345476150512695 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.734595775604248 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.73464035987854 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7346341609954834 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.7346625328063965 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.734619617462158 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7345778942108154 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7345566749572754 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.734584093093872 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.734572649002075 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.734497308731079 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7346084117889404 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.734647512435913 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7346532344818115 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.73467755317688 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.734565258026123 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.7345407009124756 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7345829010009766 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7345638275146484 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7346110343933105 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.7346465587615967 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.734619617462158 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.7345640659332275 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.734555721282959 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.734522819519043 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.734654188156128 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7345962524414062 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.734665870666504 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.734729290008545 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.73449969291687 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.734689950942993 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.734527587890625 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7346203327178955 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.7346184253692627 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.734654426574707 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.7345690727233887 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7345845699310303 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.7344248294830322 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7345709800720215 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.73453688621521 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.734551191329956 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7345805168151855 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7346229553222656 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7345821857452393 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.7345175743103027 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7345449924468994 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7346181869506836 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7346303462982178 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7345943450927734 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7345337867736816 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.734485387802124 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.734600067138672 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.73449444770813 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7345407009124756 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.734541654586792 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7346534729003906 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7345964908599854 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7346436977386475 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7345454692840576 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7345359325408936 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7345447540283203 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7346134185791016 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7346115112304688 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7346363067626953 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7344231605529785 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7347021102905273 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7345221042633057 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7345943450927734 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7345829010009766 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7345869541168213 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7346365451812744 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.734522819519043 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7346134185791016 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7346041202545166 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.734579086303711 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.734541654586792 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7345662117004395 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.734529733657837 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7344295978546143 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.734654426574707 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.734459400177002 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7346246242523193 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7345612049102783 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.734588384628296 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.734593152999878 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7344822883605957 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.7344632148742676 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7345619201660156 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7345921993255615 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7345597743988037 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7345449924468994 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.734584093093872 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.734616279602051 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.73465633392334 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7347724437713623 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.73455810546875 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.734588384628296 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7346038818359375 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.734541654586792 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7345941066741943 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.734549045562744 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.734455108642578 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.734563112258911 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.7345495223999023 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7345454692840576 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.734628677368164 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.734528064727783 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.734572649002075 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7344772815704346 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.734606981277466 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.7345762252807617 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7345902919769287 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7346091270446777 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.734644651412964 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7344255447387695 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.7345662117004395 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.734545946121216 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.734564781188965 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7345821857452393 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.734553813934326 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7345547676086426 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7345669269561768 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.7346668243408203 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.7345566749572754 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.73456072807312 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.734619617462158 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.734524965286255 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.7346556186676025 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.734452486038208 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.734555721282959 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.7345199584960938 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.734598398208618 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.7346301078796387 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7345778942108154 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.734621047973633 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.734581470489502 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.7344813346862793 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7346031665802 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.734614372253418 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7346062660217285 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.734539747238159 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.734443426132202 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.7345974445343018 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.734567165374756 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7345547676086426 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7345900535583496 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.734567165374756 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7345187664031982 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.734522819519043 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.734391927719116 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.734485626220703 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.734562873840332 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.734584093093872 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7345075607299805 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7346365451812744 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7345528602600098 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7345638275146484 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7344484329223633 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.734602451324463 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.7345774173736572 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7345266342163086 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.7345619201660156 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.7344727516174316 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.734545946121216 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.734485626220703 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.734557867050171 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7345356941223145 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7345197200775146 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.7345495223999023 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.734449625015259 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.734473466873169 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.734694719314575 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.7345573902130127 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7345080375671387 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.734482526779175 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.7345521450042725 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.734640598297119 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.734456777572632 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.7346606254577637 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.73452091217041 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.7345821857452393 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.734565258026123 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.7345566749572754 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7344915866851807 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7344703674316406 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.7345218658447266 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7345709800720215 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.734578847885132 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.7345340251922607 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.734602451324463 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.73451566696167 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.734581470489502 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.734560966491699 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.734541893005371 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.734522819519043 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7345387935638428 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7346031665802 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.734565019607544 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7347443103790283 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.7345056533813477 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7346251010894775 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.7345499992370605 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7344672679901123 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7346041202545166 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.7344632148742676 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.734426498413086 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.734578847885132 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7345762252807617 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.7345516681671143 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.734593152999878 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7346081733703613 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.734417200088501 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7346246242523193 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.7345476150512695 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.734567165374756 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.73453688621521 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.734541893005371 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.734567165374756 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.7344400882720947 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7345335483551025 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.734679698944092 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7345597743988037 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.734471082687378 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.734562873840332 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7345941066741943 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7345056533813477 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.73464035987854 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.7345731258392334 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.734574317932129 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.734539031982422 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7345516681671143 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.7345104217529297 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.734452486038208 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.734529733657837 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7344906330108643 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.7345638275146484 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.734612464904785 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.734550714492798 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.7344839572906494 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7344822883605957 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.734496593475342 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7346954345703125 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7345056533813477 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7345175743103027 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7344887256622314 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.734557867050171 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7343711853027344 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7344515323638916 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.734492778778076 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.734738826751709 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7346646785736084 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7345831394195557 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.734588384628296 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.73451828956604 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7346553802490234 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7344958782196045 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.7345378398895264 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.734598398208618 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.734602928161621 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.7345831394195557 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.73457670211792 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7345924377441406 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7346994876861572 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7345199584960938 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.734593391418457 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.734431505203247 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.734550714492798 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.734618902206421 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.7343008518218994 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7345972061157227 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7345941066741943 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7345685958862305 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7345733642578125 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7344791889190674 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.734434127807617 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.7345612049102783 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.7345805168151855 + } + ] + }, + { + "bit": 11, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7249364852905273 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7249724864959717 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7250046730041504 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.724954128265381 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.725029230117798 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.725107192993164 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.725029945373535 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.7249836921691895 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7249438762664795 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.724973201751709 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.724818706512451 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.7250115871429443 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7250165939331055 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.724968671798706 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.724959373474121 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.725015640258789 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.724928617477417 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.725060224533081 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.725005865097046 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7249128818511963 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7250638008117676 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.725050687789917 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.7249021530151367 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7250146865844727 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.7249295711517334 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.7250235080718994 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.724914312362671 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.7250828742980957 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.725083351135254 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.725010395050049 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.7250640392303467 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.725050449371338 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.725111722946167 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.7251479625701904 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.725100040435791 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.725100040435791 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.725027322769165 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.724945545196533 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7249932289123535 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7248611450195312 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7250094413757324 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.72493314743042 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.725032329559326 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.7249765396118164 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7249393463134766 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7250478267669678 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.725015640258789 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7248740196228027 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7251086235046387 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.725064754486084 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7250797748565674 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7250375747680664 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7251741886138916 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7251036167144775 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.725013494491577 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.724940538406372 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7250144481658936 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.724994421005249 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.725001335144043 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.725050449371338 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7249908447265625 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.7250757217407227 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7249443531036377 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7250990867614746 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7250027656555176 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7251791954040527 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.725084066390991 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.724944591522217 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.725067615509033 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.725050449371338 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.7251107692718506 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.72505259513855 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7250823974609375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.7251081466674805 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7249755859375 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7249996662139893 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.724839925765991 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.7250468730926514 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.725013256072998 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.7249574661254883 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.724968671798706 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.725084066390991 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7250494956970215 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7251081466674805 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.725013256072998 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7250454425811768 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.725001335144043 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.725128650665283 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.724813222885132 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.724931240081787 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7250707149505615 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.7250735759735107 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7250218391418457 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.7251498699188232 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.724989175796509 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.7250092029571533 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.7251241207122803 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.724914312362671 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7250566482543945 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7249996662139893 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.725109577178955 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.724990129470825 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7249202728271484 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7249042987823486 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7249510288238525 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7250113487243652 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7249794006347656 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.725060224533081 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.72509765625 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.7248878479003906 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.7251763343811035 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7250616550445557 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7249677181243896 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.725003719329834 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.725076913833618 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7252590656280518 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7250468730926514 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7249903678894043 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.7249696254730225 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.725001573562622 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.7249867916107178 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.725088596343994 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.725191831588745 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7250425815582275 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7250778675079346 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7250547409057617 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7250587940216064 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7250125408172607 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7250285148620605 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.725057601928711 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.7249350547790527 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.725036859512329 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.725004196166992 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7250800132751465 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.7250425815582275 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.7249722480773926 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7250335216522217 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7250335216522217 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7250378131866455 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.7250406742095947 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.72505784034729 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.7250218391418457 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7249443531036377 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.725022792816162 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.724874496459961 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7250828742980957 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.724991798400879 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7249972820281982 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7250115871429443 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.725131034851074 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.725041627883911 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.724879026412964 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.7250494956970215 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.725006341934204 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.7250144481658936 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.7250800132751465 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.725139617919922 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.7249836921691895 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7249574661254883 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.724975109100342 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7249972820281982 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.724931240081787 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7250442504882812 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.725066900253296 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.7250514030456543 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.724954128265381 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.72505259513855 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.725182294845581 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.72495174407959 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7250094413757324 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7251250743865967 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.725090503692627 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.725045919418335 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.725074529647827 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.72497820854187 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.7249701023101807 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7250325679779053 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.7251977920532227 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7249691486358643 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.725043296813965 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.725050449371338 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.724984645843506 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7250113487243652 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.725008249282837 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.7250313758850098 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.7249865531921387 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.724992513656616 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.725097894668579 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.7250216007232666 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.725013256072998 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7250397205352783 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7249233722686768 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.7249555587768555 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.7249231338500977 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.725029945373535 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.7248587608337402 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.724936008453369 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.725034713745117 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7250797748565674 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7250094413757324 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.725025177001953 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.725029945373535 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7250137329101562 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7250146865844727 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.725001573562622 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.7250306606292725 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7250537872314453 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.7249982357025146 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.7250006198883057 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.7249438762664795 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.7250285148620605 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.725022792816162 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7249090671539307 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.7249996662139893 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7248687744140625 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.7251036167144775 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.724978446960449 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.725088596343994 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.7249643802642822 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.724961519241333 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7250025272369385 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.724973201751709 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.725100040435791 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7250194549560547 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.725006341934204 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7250406742095947 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.7251405715942383 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.7251529693603516 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.72491192817688 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.724968194961548 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.725031614303589 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7250185012817383 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7250537872314453 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7250638008117676 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.7250990867614746 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.725020408630371 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.7250285148620605 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.725109577178955 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.7250759601593018 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.724940061569214 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.72512149810791 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.725017547607422 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7250447273254395 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7249882221221924 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7250218391418457 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.724963426589966 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.725088119506836 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7250850200653076 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.725043773651123 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.7250051498413086 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7248268127441406 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7250025272369385 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.725020408630371 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.725013256072998 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.724925994873047 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.7249908447265625 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.725013494491577 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.7250442504882812 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7250187397003174 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.7250077724456787 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7250728607177734 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7250757217407227 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.725029468536377 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.724945545196533 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.7249865531921387 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.724886417388916 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.7249534130096436 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.7250282764434814 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.724982738494873 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.725076913833618 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7251136302948 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.7250325679779053 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.724940299987793 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.724921226501465 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.725076913833618 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.7250123023986816 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.7249152660369873 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.724938154220581 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.7249064445495605 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7249855995178223 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7249350547790527 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.7250237464904785 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.72493839263916 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.724973201751709 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7250454425811768 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7249953746795654 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.724971055984497 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.724959135055542 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.7250595092773438 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.725080728530884 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7250051498413086 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.725060224533081 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7250325679779053 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7249703407287598 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7250499725341797 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.724957227706909 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.724963426589966 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7249703407287598 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.724881887435913 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.7250325679779053 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7249321937561035 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.724987745285034 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.725071668624878 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.725006341934204 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.7249443531036377 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.7249021530151367 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7249867916107178 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.724916934967041 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7250428199768066 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7250537872314453 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7250800132751465 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.724963426589966 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.724949598312378 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.7250237464904785 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7250277996063232 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.725001573562622 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.7250077724456787 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.725052833557129 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.725083351135254 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7249743938446045 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.725062370300293 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.724982738494873 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7249677181243896 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.7250282764434814 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.724994659423828 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.725084066390991 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.724886178970337 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.7249491214752197 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.725031614303589 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7250092029571533 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7250187397003174 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.725004196166992 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.724963426589966 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.725071668624878 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7248799800872803 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7249772548675537 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7249653339385986 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.7250242233276367 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7250077724456787 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.724978446960449 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7249152660369873 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7250194549560547 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7249486446380615 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7250113487243652 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.725057601928711 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7249813079833984 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7250378131866455 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7250473499298096 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7249491214752197 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7249581813812256 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7250094413757324 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7250478267669678 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7250216007232666 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7250418663024902 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.725001573562622 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.725008726119995 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7248940467834473 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.72493839263916 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.725043773651123 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7249982357025146 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.724975109100342 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.724907398223877 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.724832773208618 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.725123167037964 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7250680923461914 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.724961280822754 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7250263690948486 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.725045919418335 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7250707149505615 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.7249932289123535 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.724977493286133 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7250523567199707 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.724978446960449 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7250068187713623 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.7249233722686768 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7250003814697266 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.7248969078063965 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7248106002807617 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7250378131866455 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.725013256072998 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7250123023986816 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.724978446960449 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.724870204925537 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.725050449371338 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.7250919342041016 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.724982261657715 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.7249717712402344 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7250263690948486 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.724907398223877 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.7249722480773926 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.725107192993164 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.724957227706909 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.725015640258789 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.7249794006347656 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7249701023101807 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.725001335144043 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.7249743938446045 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7249643802642822 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.7250757217407227 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7249844074249268 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.724978446960449 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7249972820281982 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.725083827972412 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7250163555145264 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7250547409057617 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.724766731262207 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.724998712539673 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7249984741210938 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7249813079833984 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.7249135971069336 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.724928617477417 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.7250468730926514 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.7250478267669678 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.7250514030456543 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.7249755859375 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.7250137329101562 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7250137329101562 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7249062061309814 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7249836921691895 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.725074529647827 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7250218391418457 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.725026845932007 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.725053548812866 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.725017786026001 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.7250094413757324 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.7249982357025146 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.7250335216522217 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7250442504882812 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7249813079833984 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.7249391078948975 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7250003814697266 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7250447273254395 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.725043773651123 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.7250916957855225 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.725029468536377 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.724994421005249 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.725029230117798 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7249510288238525 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.724898099899292 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.72505784034729 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7249984741210938 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.725013256072998 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.7250094413757324 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.724987268447876 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.725060224533081 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.725015640258789 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7249350547790527 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.724994421005249 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.725036859512329 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7249722480773926 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.725003480911255 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.7249982357025146 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.724936008453369 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.724996328353882 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.724982261657715 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.724989175796509 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7250282764434814 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.7250235080718994 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.725001573562622 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7248899936676025 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.7250800132751465 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.724945545196533 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7250704765319824 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.724947929382324 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7249934673309326 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.724992513656616 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.725022792816162 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7249162197113037 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.725004196166992 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.724998712539673 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.725001573562622 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.7250778675079346 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.7251029014587402 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.72493052482605 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.724963665008545 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7251055240631104 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.7249865531921387 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7250523567199707 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7249908447265625 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.724996566772461 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7249653339385986 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7249562740325928 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.7249908447265625 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7250356674194336 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.7250537872314453 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.725083351135254 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.724994659423828 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.7250828742980957 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.7251758575439453 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.724950075149536 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7249643802642822 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.7250123023986816 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.724954128265381 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7249996662139893 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.724949598312378 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7250776290893555 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.7250051498413086 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.724957227706909 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7249913215637207 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.724954128265381 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.725043296813965 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.72509765625 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7249972820281982 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.7249011993408203 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7249438762664795 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7250194549560547 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.7249908447265625 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.724973440170288 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7249553203582764 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7249755859375 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.7250680923461914 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.7250218391418457 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.7249996662139893 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.725027322769165 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.7249953746795654 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.7250638008117676 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.7249510288238525 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7250235080718994 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.7249765396118164 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.724959135055542 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.7249159812927246 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.7251105308532715 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7249224185943604 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.724987268447876 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7249982357025146 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.725015640258789 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7249596118927 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.724945545196533 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7250823974609375 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.724799633026123 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7249159812927246 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.724989175796509 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.724928617477417 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7250986099243164 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7250123023986816 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.7250077724456787 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.724881887435913 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7248778343200684 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.725043296813965 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.724968671798706 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.725050687789917 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.725094795227051 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.725027322769165 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.7248761653900146 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7250664234161377 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.724896192550659 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7250547409057617 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7249042987823486 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.724940299987793 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.7249596118927 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.7248661518096924 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.72513747215271 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.724942207336426 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7250003814697266 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7249953746795654 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.724931240081787 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7249796390533447 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.725111722946167 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.725125551223755 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.7250845432281494 + } + ] + }, + { + "bit": 12, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.717428207397461 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.717423915863037 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7174558639526367 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.717453956604004 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.717428207397461 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.7174930572509766 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7174088954925537 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.717412233352661 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7174131870269775 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.717503786087036 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.717482089996338 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.717480182647705 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7174408435821533 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.717602014541626 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7173964977264404 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7174928188323975 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.71745228767395 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7173116207122803 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.717398166656494 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7175400257110596 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7173357009887695 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.71730899810791 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.7175168991088867 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7174417972564697 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.717445135116577 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.7173919677734375 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.7174646854400635 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.717289924621582 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.717362880706787 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7174570560455322 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.7174031734466553 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.717291831970215 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.7173688411712646 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.7172648906707764 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.717392683029175 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.717336893081665 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7174363136291504 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.717363119125366 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7174110412597656 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.71749210357666 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.717418670654297 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.717302083969116 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.717501163482666 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.7174646854400635 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.717535972595215 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.717539072036743 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7174973487854004 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7173378467559814 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.717395782470703 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7173149585723877 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7174618244171143 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7174832820892334 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7173566818237305 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.717479944229126 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.717445135116577 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.71748685836792 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.7175443172454834 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.717435359954834 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.71746563911438 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7173972129821777 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.717350721359253 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7173352241516113 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.717374086380005 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7174110412597656 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7173779010772705 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.717341661453247 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7173192501068115 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.71749210357666 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.7174742221832275 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.7173781394958496 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.7174291610717773 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7173681259155273 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.71740984916687 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7174038887023926 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7173876762390137 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.7175452709198 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.7174248695373535 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7173867225646973 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.717472791671753 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.717423915863037 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.717404365539551 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.717421054840088 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7173619270324707 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.717411994934082 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7174389362335205 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.7175018787384033 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7174108028411865 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7173526287078857 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.7174601554870605 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7173831462860107 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.7174670696258545 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7174108028411865 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.717303514480591 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7174429893493652 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.717442750930786 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.717364549636841 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7175283432006836 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7174508571624756 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7174079418182373 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7174909114837646 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.7173421382904053 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7174272537231445 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7174079418182373 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7173025608062744 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7174479961395264 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7174651622772217 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7174506187438965 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.7174673080444336 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.717416763305664 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.7173428535461426 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.717556953430176 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.717439889907837 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.7174859046936035 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7174441814422607 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7174298763275146 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7173311710357666 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.7174720764160156 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.717460870742798 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.717376947402954 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.71742582321167 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.717360019683838 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.717331886291504 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7173779010772705 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.717374086380005 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7174363136291504 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7174060344696045 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7174465656280518 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7174038887023926 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.7174553871154785 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7173900604248047 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.7173001766204834 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.717484951019287 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.71738862991333 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.71748423576355 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7174389362335205 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7172420024871826 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.717418909072876 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.717412233352661 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.71747088432312 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.717412233352661 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7175281047821045 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.717407703399658 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.717519998550415 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.717445135116577 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.717421770095825 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7174720764160156 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7173821926116943 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7174177169799805 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.717428207397461 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.717409133911133 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.7175350189208984 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.717477798461914 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.717468738555908 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.7174320220947266 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7174108028411865 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7174479961395264 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7175402641296387 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7173969745635986 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.717424154281616 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.7173268795013428 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.71748685836792 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.7175278663635254 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.717514753341675 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.71740984916687 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.717423915863037 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7173941135406494 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.717360019683838 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.7173397541046143 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.717532157897949 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.717289924621582 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.71750807762146 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.7174036502838135 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7173969745635986 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.7173469066619873 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7174081802368164 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.7173080444335938 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7174036502838135 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7175190448760986 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7174651622772217 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7174203395843506 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.717416763305664 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.7174508571624756 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7173449993133545 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7172768115997314 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.71738862991333 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7173805236816406 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.717381238937378 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.717489004135132 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.717385768890381 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.7174599170684814 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7173569202423096 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.7175090312957764 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.71744704246521 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7174131870269775 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.717423915863037 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7174267768859863 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.7175912857055664 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.717482328414917 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.717564821243286 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7174088954925537 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.7174150943756104 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.7173681259155273 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.717435836791992 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.7174789905548096 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.7173478603363037 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.717505931854248 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.71745228767395 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.717360019683838 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.717505931854248 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.7174816131591797 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7174017429351807 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.7173638343811035 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7174160480499268 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.7174479961395264 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.7174887657165527 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.71742582321167 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7174720764160156 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7175371646881104 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.7174108028411865 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7173972129821777 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.7174832820892334 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.717439889907837 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.717454433441162 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.7174692153930664 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.7173290252685547 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7174038887023926 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.7173662185668945 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7174291610717773 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.717405319213867 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.71740984916687 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.717435359954834 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.7173919677734375 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.717480182647705 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.717437267303467 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.7174887657165527 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.7174408435821533 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.71748423576355 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7173025608062744 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7174270153045654 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.717494249343872 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7173850536346436 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.717453718185425 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7174487113952637 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7175028324127197 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.71744966506958 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.717501163482666 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7174229621887207 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.717475175857544 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7174394130706787 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.717421054840088 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7175533771514893 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.7173779010772705 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.717344045639038 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.7173681259155273 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.717348098754883 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.717406988143921 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7173991203308105 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7175228595733643 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.717377185821533 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.71750545501709 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.7174322605133057 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7175559997558594 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.7174816131591797 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.7174229621887207 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.717405319213867 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.71745228767395 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.717302083969116 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.717334508895874 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.7173876762390137 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.7173407077789307 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.7174441814422607 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.7174742221832275 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.717552900314331 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7174341678619385 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.717557907104492 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.717433214187622 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7174341678619385 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.717348575592041 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7173991203308105 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7174017429351807 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7174761295318604 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7174007892608643 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7174487113952637 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.7175002098083496 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.7174782752990723 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7174441814422607 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7174432277679443 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.717411756515503 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.71730375289917 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7173638343811035 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7174553871154785 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.7173964977264404 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7174482345581055 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7174367904663086 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7174155712127686 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.717331886291504 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7174527645111084 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7173829078674316 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7174620628356934 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.717459201812744 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.7175133228302 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.717484951019287 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7175002098083496 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.717445135116577 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7174267768859863 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.71740984916687 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.717499256134033 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.7174341678619385 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.7173666954040527 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.717461109161377 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.717461109161377 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.717506170272827 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.7174713611602783 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.717412233352661 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.717437982559204 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7175817489624023 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.717459201812744 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7174527645111084 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.717433214187622 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.7174620628356934 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7174432277679443 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7174620628356934 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.7175047397613525 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.717453956604004 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7174439430236816 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.717398166656494 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.717484951019287 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.71736216545105 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7174506187438965 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.717501163482666 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.71743106842041 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7174301147460938 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.7173948287963867 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.717411756515503 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.717470407485962 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7173430919647217 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.717456102371216 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.717459201812744 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7175018787384033 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7174108028411865 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7173805236816406 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.717392921447754 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7174150943756104 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7175278663635254 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7173876762390137 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7173707485198975 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7173995971679688 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7174057960510254 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7174320220947266 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.717542886734009 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7174782752990723 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7174389362335205 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7174272537231445 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.71744966506958 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.717374086380005 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.7174336910247803 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7175071239471436 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.7175498008728027 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.717310667037964 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7174229621887207 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7174417972564697 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.717392921447754 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.717400312423706 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.71744704246521 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.717358112335205 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.717341661453247 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.717439889907837 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.717391014099121 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7173850536346436 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.717424154281616 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.717400074005127 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.7173964977264404 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.717350721359253 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7174599170684814 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7173867225646973 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7174673080444336 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.7174160480499268 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7174007892608643 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7174291610717773 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.717472791671753 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.7174532413482666 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.7174267768859863 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7174220085144043 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.7174739837646484 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.717363119125366 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.717357635498047 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.7173590660095215 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.7173893451690674 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7174313068389893 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7173850536346436 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.7175164222717285 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.71738862991333 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.717355728149414 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.71744704246521 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7174673080444336 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7173960208892822 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.7174272537231445 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7173781394958496 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7174558639526367 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.7175400257110596 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.7174203395843506 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7173876762390137 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.717414617538452 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.7174601554870605 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.7173707485198975 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.7172725200653076 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.717501401901245 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.7174131870269775 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.7174291610717773 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.7174108028411865 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.717454433441162 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7175347805023193 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7175369262695312 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.7173688411712646 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7174057960510254 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.7174057960510254 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.717388868331909 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7173867225646973 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.7174363136291504 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.7175281047821045 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.71743106842041 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.717418670654297 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7174301147460938 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.7174160480499268 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.717456817626953 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7173900604248047 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.717406749725342 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.717352867126465 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.7174322605133057 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7173821926116943 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7174417972564697 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.717428207397461 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7174506187438965 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7174196243286133 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7173867225646973 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.717435836791992 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.7173819541931152 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.717424154281616 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.717453956604004 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.7173290252685547 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.717296838760376 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.7174220085144043 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.7174429893493652 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7173731327056885 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7174172401428223 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.717459201812744 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.717474937438965 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7173900604248047 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.717406749725342 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.7174110412597656 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.71745228767395 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.717453956604004 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.717391014099121 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7173819541931152 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.7174289226531982 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.71746826171875 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.717406749725342 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.7174177169799805 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.71740984916687 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.717472791671753 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.717289924621582 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7174267768859863 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.7174692153930664 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7174150943756104 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7174298763275146 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.717437267303467 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.717386245727539 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.717406988143921 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.7173707485198975 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7174336910247803 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.7173948287963867 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.717437982559204 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.717405319213867 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.717475175857544 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7174553871154785 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7175133228302 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.7173407077789307 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7175240516662598 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.7174389362335205 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.717466354370117 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7174854278564453 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.717390775680542 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.7173280715942383 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.7174811363220215 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.717362880706787 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.7174196243286133 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.717400074005127 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.71749210357666 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.71738862991333 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7173876762390137 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.717512369155884 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7174808979034424 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.717437744140625 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.7175400257110596 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7173938751220703 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.7174389362335205 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7174642086029053 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.717456817626953 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7174441814422607 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7174088954925537 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.7174460887908936 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7174036502838135 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.71743106842041 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7174346446990967 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.717306613922119 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.717413902282715 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.7174460887908936 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.717404365539551 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.71744966506958 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.717315912246704 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.7174699306488037 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7173867225646973 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.7174482345581055 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.7174267768859863 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.717496633529663 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.717360019683838 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7174079418182373 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7173290252685547 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.717360019683838 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7174367904663086 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7173707485198975 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.71744704246521 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7173779010772705 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.717418909072876 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7173588275909424 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.717468738555908 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.717406988143921 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.717458963394165 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.717364549636841 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.7174203395843506 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.7173900604248047 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7173538208007812 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.717482328414917 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.7174670696258545 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.717411756515503 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.71743106842041 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.717355728149414 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7173831462860107 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7174582481384277 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.717392921447754 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7174301147460938 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.717406988143921 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.7174177169799805 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.7173078060150146 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.7173166275024414 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7175278663635254 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7174577713012695 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.717449188232422 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.717402696609497 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7174203395843506 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7173171043395996 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.717336654663086 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.717519760131836 + } + ] + }, + { + "bit": 13, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.7112629413604736 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7113375663757324 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.711299419403076 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7113418579101562 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.7112488746643066 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.7113454341888428 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7113637924194336 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.7112746238708496 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.711216926574707 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.711273193359375 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.711271047592163 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.711357831954956 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7113001346588135 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.711317777633667 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7113728523254395 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7113733291625977 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.7112679481506348 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.711397886276245 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.7113540172576904 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7111804485321045 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7113988399505615 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.711350917816162 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.711249828338623 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7112538814544678 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.71136212348938 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.711294651031494 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.711405038833618 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.711427927017212 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7113466262817383 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7113196849823 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.711432933807373 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.7114906311035156 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.711319923400879 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.711472749710083 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.711350679397583 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.711390733718872 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7113161087036133 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7113144397735596 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7112598419189453 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7111480236053467 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7113869190216064 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.711230993270874 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.71134352684021 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.711362838745117 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.711254119873047 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7113358974456787 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.7113037109375 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7112159729003906 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7112700939178467 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.7113709449768066 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7113454341888428 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7113215923309326 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7113938331604004 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7113945484161377 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.7113289833068848 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7113547325134277 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.711338520050049 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.71132755279541 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.7113778591156006 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7112390995025635 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.711324691772461 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.711244583129883 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.711254119873047 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7113428115844727 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.711336851119995 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.711338520050049 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.711455821990967 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.711336851119995 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.7112884521484375 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.711366653442383 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.711329460144043 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.7113454341888428 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.711336612701416 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.71134090423584 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7114405632019043 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7113037109375 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.71130108833313 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.7113289833068848 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.711289644241333 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.7112619876861572 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.7113263607025146 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.711354970932007 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7113046646118164 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7112348079681396 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.7113449573516846 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7112808227539062 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.711423397064209 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.711297035217285 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7112984657287598 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.711313486099243 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7112619876861572 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.7113728523254395 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7112843990325928 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.71140456199646 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.711350679397583 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.711371898651123 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.7113728523254395 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7111878395080566 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7113258838653564 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7113685607910156 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7113938331604004 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.711357831954956 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7114009857177734 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.7113988399505615 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.711324691772461 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7113707065582275 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7113230228424072 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7113258838653564 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.711289644241333 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.711324691772461 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.7113564014434814 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.711271047592163 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.711390495300293 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.711310863494873 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.711249828338623 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7113215923309326 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7112605571746826 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7114388942718506 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.71136212348938 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.711392879486084 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.711296796798706 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.711326837539673 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.7113606929779053 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7113595008850098 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7113635540008545 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.711265802383423 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7112886905670166 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7112843990325928 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.711329460144043 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.71128249168396 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.7111878395080566 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7112905979156494 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.71140456199646 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7113568782806396 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.711315393447876 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.711322784423828 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7113444805145264 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7113733291625977 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7112560272216797 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.71122670173645 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.711310863494873 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.7113239765167236 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7112948894500732 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.711374521255493 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7113616466522217 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7113475799560547 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.711254119873047 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7112927436828613 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7113027572631836 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.711268424987793 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7113168239593506 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.711245536804199 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.7113842964172363 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.7113656997680664 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.7113723754882812 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.71138072013855 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.711252450942993 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.7113306522369385 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7113535404205322 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7113280296325684 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7112231254577637 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7114267349243164 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7113335132598877 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.7113168239593506 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.7113986015319824 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.7112081050872803 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.711299419403076 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7112605571746826 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.71128249168396 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.711327075958252 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7113559246063232 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.711296796798706 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.711376428604126 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.7114779949188232 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.711313009262085 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.7113237380981445 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7114098072052 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.711454391479492 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.711369752883911 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.71138072013855 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.711345672607422 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7112948894500732 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7112605571746826 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7112860679626465 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.711306571960449 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.711346387863159 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7114417552948 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7114789485931396 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.7113754749298096 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7113656997680664 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7113311290740967 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7113540172576904 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.7113378047943115 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.711233615875244 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7113311290740967 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.7113161087036133 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.7112927436828613 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.711357831954956 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7114386558532715 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.71138596534729 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.7112576961517334 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.7112884521484375 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7112696170806885 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7112796306610107 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.711350440979004 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.711338996887207 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7113311290740967 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.711249828338623 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.711425542831421 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.711387872695923 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.71136212348938 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.7112958431243896 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.711320638656616 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.7112886905670166 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.711336612701416 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.7113375663757324 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7113699913024902 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.711352586746216 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.711305856704712 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.71136212348938 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.711216926574707 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7112975120544434 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.7112958431243896 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.711366653442383 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.71134090423584 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.711338996887207 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.7114319801330566 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.7113349437713623 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.7112619876861572 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7113037109375 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.711395502090454 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7113497257232666 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7114028930664062 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7114064693450928 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.7113125324249268 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.711392402648926 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.7112224102020264 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7113869190216064 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.7113399505615234 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.711407423019409 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.711315393447876 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7114126682281494 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7113378047943115 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7112839221954346 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.711392879486084 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7113285064697266 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.711397647857666 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7114009857177734 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7113847732543945 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.7113776206970215 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.711271047592163 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7113304138183594 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7113819122314453 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.7113356590270996 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7113258838653564 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.7113559246063232 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.71136212348938 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.711387872695923 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7113096714019775 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.7113585472106934 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.711305856704712 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.711315393447876 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.711313486099243 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.711254596710205 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.711345672607422 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7112679481506348 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.7113637924194336 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.71132755279541 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.7113842964172363 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.7114248275756836 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.711432933807373 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.711306571960449 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.711261510848999 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.711364507675171 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.711338758468628 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.7113161087036133 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.7112843990325928 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7112538814544678 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.7113454341888428 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.71132755279541 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7113428115844727 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.7114386558532715 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.7112667560577393 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7113001346588135 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.711216926574707 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.711366653442383 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7112436294555664 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.71127986907959 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.711308717727661 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7113089561462402 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.71140193939209 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.711352586746216 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.711385726928711 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7112159729003906 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.711392879486084 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.7113518714904785 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.711418867111206 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.711338520050049 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7112338542938232 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.711359977722168 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.711212635040283 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.711390495300293 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7112984657287598 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.711362838745117 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.7113037109375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.71134877204895 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.711310386657715 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.711174726486206 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.711352586746216 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7113656997680664 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7113311290740967 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.7113778591156006 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.7113230228424072 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.7113001346588135 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7114856243133545 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7113935947418213 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.71134352684021 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.711397647857666 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.7113707065582275 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.711338520050049 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.7113797664642334 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.71132755279541 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.711322546005249 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.711329460144043 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7113215923309326 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7114152908325195 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7112319469451904 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.711235761642456 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.711394786834717 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.711315393447876 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.711392879486084 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7113595008850098 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.711366653442383 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7113585472106934 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7112877368927 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7113335132598877 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7113378047943115 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.711333751678467 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7113678455352783 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.711369276046753 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7113194465637207 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7113118171691895 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.711322546005249 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7112996578216553 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.711357355117798 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.711336612701416 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.711305856704712 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.711381673812866 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7111780643463135 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.711317539215088 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.711357831954956 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7113709449768066 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7113797664642334 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7113258838653564 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.711334466934204 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.711246967315674 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.711315393447876 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.711266040802002 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7113616466522217 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7113418579101562 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.7113585472106934 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.711230754852295 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.711291551589966 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7113280296325684 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7113037109375 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7113263607025146 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.711344003677368 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.711320638656616 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7111868858337402 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.7113709449768066 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7114224433898926 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7112700939178467 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7113089561462402 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7113077640533447 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.711317777633667 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7113614082336426 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.711296796798706 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.711346387863159 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.711350679397583 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7112796306610107 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7113196849823 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.7113723754882812 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7112767696380615 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7113637924194336 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.711357355117798 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.711357831954956 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.711331605911255 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.71128249168396 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.7113311290740967 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.7112667560577393 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.7113099098205566 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.711301565170288 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.711381673812866 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.711338996887207 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.711322784423828 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.711371898651123 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.711148977279663 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7113075256347656 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.7113535404205322 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7113804817199707 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.711336612701416 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7113125324249268 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.711306571960449 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.711357831954956 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7113311290740967 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.711235761642456 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.711313486099243 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7113027572631836 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.711296796798706 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.711291551589966 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.711244583129883 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.7114548683166504 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.711306571960449 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.711315393447876 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.7112860679626465 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.711315393447876 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7113654613494873 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7112317085266113 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.711177110671997 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.7112925052642822 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7113027572631836 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.711317539215088 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7113516330718994 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7113113403320312 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.711183547973633 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.711243152618408 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.7113046646118164 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7112877368927 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.71128511428833 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.7113239765167236 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.711275815963745 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.71125864982605 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.711305856704712 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.7113914489746094 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.7113471031188965 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7112925052642822 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7113418579101562 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7113454341888428 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7112228870391846 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7114057540893555 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7113547325134277 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.7112691402435303 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.711293935775757 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.711324691772461 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.7113258838653564 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.7113828659057617 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.711299419403076 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.7113113403320312 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.7112700939178467 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.711293935775757 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.711305618286133 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.711344003677368 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.7112488746643066 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7112295627593994 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.711268424987793 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.7113049030303955 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.711275577545166 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.7112770080566406 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.711258888244629 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.711313009262085 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.711376428604126 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.711296796798706 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7113430500030518 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.711277723312378 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7113230228424072 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.7113218307495117 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7112808227539062 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7113263607025146 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.7112507820129395 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.711275815963745 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7113304138183594 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.711268424987793 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.711326837539673 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.7113096714019775 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.711298942565918 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.711193084716797 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.7113590240478516 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.711308002471924 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.711289644241333 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7112908363342285 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.71124267578125 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.711151599884033 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.711296796798706 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7113559246063232 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.71132493019104 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7113475799560547 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7113068103790283 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.7112128734588623 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.7113850116729736 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.711334705352783 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.711317539215088 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.711308717727661 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7112958431243896 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7112605571746826 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7112905979156494 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7113356590270996 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.7113196849823 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.711254119873047 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.711308002471924 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.711275815963745 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7113089561462402 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.7113535404205322 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7113161087036133 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.7113161087036133 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.711331844329834 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7113187313079834 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.711291551589966 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.71134352684021 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.711181640625 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7112576961517334 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.711291551589966 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.7113239765167236 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.711230993270874 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7113099098205566 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.711308717727661 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.711280345916748 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.7112834453582764 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7113754749298096 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.711291790008545 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.7112843990325928 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.7112107276916504 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.7113125324249268 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.711155891418457 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7112576961517334 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.711371898651123 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7113001346588135 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.711293935775757 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7112374305725098 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.711308002471924 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7110986709594727 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7112112045288086 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.7112929821014404 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.7112860679626465 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7113258838653564 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.711313486099243 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.7113516330718994 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.7111728191375732 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.711399793624878 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7113258838653564 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.71134352684021 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.7113215923309326 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.7113516330718994 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.7112739086151123 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.7112555503845215 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.711338520050049 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.711261749267578 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7112929821014404 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7112619876861572 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.711299419403076 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.7112691402435303 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.7113027572631836 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.7113993167877197 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7111568450927734 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7112960815429688 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7113037109375 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7112996578216553 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.711338520050049 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7113890647888184 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.711416006088257 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.711251974105835 + } + ] + }, + { + "bit": 14, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.705030679702759 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.704963445663452 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7049667835235596 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7050507068634033 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.7049453258514404 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.7050507068634033 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7050368785858154 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.7049560546875 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7050230503082275 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.704890489578247 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7049055099487305 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.7049429416656494 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7050576210021973 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.7049763202667236 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.704965829849243 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7049312591552734 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.7050068378448486 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7049617767333984 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.704883098602295 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.704987049102783 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.704927921295166 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.704954147338867 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.705036163330078 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.705009937286377 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.7049927711486816 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.705064058303833 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.704881191253662 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.7048676013946533 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7049400806427 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.705012798309326 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.7049152851104736 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.704878330230713 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.7050442695617676 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.70483136177063 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.7049481868743896 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.7049858570098877 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.705015182495117 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.704890012741089 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7049851417541504 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.705029010772705 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7049453258514404 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.704960346221924 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.7050251960754395 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.704907178878784 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.705042600631714 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7050020694732666 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.704986333847046 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7050440311431885 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7049667835235596 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.7049100399017334 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7049710750579834 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7049999237060547 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.704961061477661 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.704986095428467 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.704930305480957 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7049412727355957 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7049622535705566 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.704930305480957 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.705038070678711 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7049481868743896 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7049622535705566 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.7049190998077393 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7049190998077393 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7049779891967773 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.704936981201172 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7050468921661377 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.704942226409912 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7048943042755127 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.7050020694732666 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.704993963241577 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.7050678730010986 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.704996109008789 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7050156593322754 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.704996109008789 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.704941987991333 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.70489239692688 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.705033779144287 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.705057144165039 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7050328254699707 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.704983949661255 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.7049999237060547 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.70497989654541 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7050576210021973 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7050368785858154 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.7049949169158936 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7049472332000732 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.704838514328003 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.705021381378174 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.704909324645996 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.705049753189087 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7050132751464844 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.704986095428467 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.70499587059021 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.7048890590667725 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7049667835235596 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.704911947250366 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.7049648761749268 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7050840854644775 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7049880027770996 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7050271034240723 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.704963445663452 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.704983949661255 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7049570083618164 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.704864263534546 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7049052715301514 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.704969882965088 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7050275802612305 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.704935312271118 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.7050509452819824 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.705021381378174 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.70487117767334 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.705008029937744 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7049779891967773 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.7049612998962402 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.704965829849243 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7049763202667236 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.70500111579895 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7048511505126953 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.7050349712371826 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.705077886581421 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.704988718032837 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.7049400806427 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.704935312271118 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.704960346221924 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7048580646514893 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7049732208251953 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7049851417541504 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7049949169158936 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7049481868743896 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.705019950866699 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.7049789428710938 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7049930095672607 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.704908847808838 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.704993963241577 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.7049763202667236 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.704982280731201 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7049829959869385 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7049264907836914 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7049741744995117 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.704949378967285 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.7049343585968018 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.704993963241577 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7050461769104004 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.704906463623047 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7049524784088135 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.70489239692688 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.705099582672119 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7049648761749268 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7048959732055664 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.705015182495117 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7049612998962402 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7050788402557373 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.7049560546875 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.70503568649292 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.7050411701202393 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.705002784729004 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.7050647735595703 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.704968214035034 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7050585746765137 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.704963445663452 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.705122947692871 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.705052137374878 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.704972982406616 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.7050015926361084 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.704993724822998 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.7049529552459717 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.7050070762634277 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7049009799957275 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.705017328262329 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.70497727394104 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.704989194869995 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.704824447631836 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.704968214035034 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.7050108909606934 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.7050108909606934 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.704944372177124 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7049551010131836 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.704942226409912 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7050442695617676 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.70493483543396 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7049453258514404 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7049498558044434 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7049453258514404 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7050297260284424 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.70501971244812 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.7050182819366455 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.7049639225006104 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7048001289367676 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.7049591541290283 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7050411701202393 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7049405574798584 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7050421237945557 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.705002784729004 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.7049930095672607 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7049591541290283 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.70500111579895 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.704916000366211 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7050037384033203 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7049100399017334 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7049832344055176 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.705054521560669 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.704970121383667 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7050797939300537 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7049710750579834 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.7049779891967773 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.7049639225006104 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7050681114196777 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.705012798309326 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.7049384117126465 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.705075979232788 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.7049808502197266 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.7049553394317627 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7050678730010986 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.70504093170166 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.704984188079834 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.7049412727355957 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7049498558044434 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.7049977779388428 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.7049639225006104 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.705026865005493 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7050716876983643 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7049214839935303 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.704960346221924 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7049593925476074 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.704963207244873 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.70491361618042 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.7049074172973633 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.7049560546875 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.704965353012085 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.704834222793579 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.7048699855804443 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.704972982406616 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7049648761749268 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.704944372177124 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.7049808502197266 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.704925060272217 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.705033779144287 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.704900026321411 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.704951524734497 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.7049341201782227 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7050390243530273 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.704906463623047 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.704918146133423 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.705029249191284 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.704927921295166 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7049918174743652 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7049949169158936 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7049918174743652 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.704960346221924 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.704981803894043 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7049238681793213 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7050585746765137 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.70500111579895 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.705028533935547 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7050187587738037 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.705008029937744 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.7048792839050293 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.7049310207366943 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7049171924591064 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.7049331665039062 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.704979419708252 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.704970121383667 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.7050275802612305 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.7050769329071045 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.705026149749756 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7050576210021973 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.704925060272217 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.7050092220306396 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.704911947250366 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.7050559520721436 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.70497465133667 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.705031156539917 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.7049429416656494 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.704890489578247 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.704970121383667 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.70497989654541 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.705054998397827 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7049968242645264 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.705064058303833 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7050108909606934 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7049171924591064 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.704981803894043 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.704998254776001 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7049872875213623 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7050161361694336 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7049334049224854 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.704989194869995 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.704951524734497 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.7050063610076904 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7050929069519043 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.704982280731201 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.7049808502197266 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7049648761749268 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7050015926361084 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.704986095428467 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.705005168914795 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7049720287323 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.705068826675415 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7049930095672607 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.704982042312622 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7050318717956543 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7049431800842285 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.704939126968384 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.705033779144287 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.705014228820801 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.705069065093994 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.705036163330078 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.705019235610962 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.704993963241577 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.704982042312622 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7050259113311768 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.7049901485443115 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.704860210418701 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.704911470413208 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7049169540405273 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7049851417541504 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.7049002647399902 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.705045223236084 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.704890012741089 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7049291133880615 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.704926013946533 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7048604488372803 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7049622535705566 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.704993724822998 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7050042152404785 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7049903869628906 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7048392295837402 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.704996109008789 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.70491099357605 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.705037832260132 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7050509452819824 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7050178050994873 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7049612998962402 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7049753665924072 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.704954147338867 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7049708366394043 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.705059051513672 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.705000877380371 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7049903869628906 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7049779891967773 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7049319744110107 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.705021381378174 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.704949378967285 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7049977779388428 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.704965829849243 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7049570083618164 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7049758434295654 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7049903869628906 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7050490379333496 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7048943042755127 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7050302028656006 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.704998254776001 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.704996109008789 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.704951524734497 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.7049524784088135 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7050249576568604 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7049918174743652 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7049357891082764 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7050249576568604 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7049753665924072 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.7049593925476074 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.705007791519165 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.704957962036133 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7049384117126465 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7050302028656006 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7049880027770996 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.704987049102783 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.704986333847046 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7049498558044434 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.7048685550689697 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7049570083618164 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.705019235610962 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.704976797103882 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7049591541290283 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.705038070678711 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7050228118896484 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.7050063610076904 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7049622535705566 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7050087451934814 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.705082893371582 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.70501971244812 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.704986095428467 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7050509452819824 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.705014228820801 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.704998016357422 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.70499587059021 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.7049593925476074 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7049810886383057 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.7050046920776367 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.704988718032837 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.704967498779297 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.704894542694092 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.705049753189087 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.7050271034240723 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.7049648761749268 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.705043077468872 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.7049951553344727 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7049100399017334 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.70500111579895 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.704981803894043 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7049591541290283 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7049880027770996 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.704984188079834 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.704998254776001 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7049667835235596 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.7050390243530273 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.705034017562866 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.7050418853759766 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7049612998962402 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.705017328262329 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.7050328254699707 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.704895257949829 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.705022096633911 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.705038070678711 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.705049753189087 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.705033779144287 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7049460411071777 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.704954147338867 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7050769329071045 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.7049202919006348 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7050037384033203 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.7049639225006104 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7049713134765625 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7050063610076904 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.7049622535705566 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.704968214035034 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.7048332691192627 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.70503568649292 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.70495343208313 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.705009937286377 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7049472332000732 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7049384117126465 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.7050089836120605 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.704927921295166 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.70500111579895 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.705066680908203 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.705005645751953 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7049882411956787 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7049808502197266 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.705019235610962 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7050328254699707 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.704989194869995 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.704965829849243 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.704988718032837 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.705091714859009 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.7048866748809814 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7049100399017334 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.70501708984375 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.7050228118896484 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.7050046920776367 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.705049753189087 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.704991340637207 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.704935312271118 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7049880027770996 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.7050347328186035 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.704986095428467 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7049896717071533 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.7050440311431885 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.7050118446350098 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7049384117126465 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.7049648761749268 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.705021381378174 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.704962730407715 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.705005645751953 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7050461769104004 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.704991340637207 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7049031257629395 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7049171924591064 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.705005645751953 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7049896717071533 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7049977779388428 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.7049713134765625 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.705014228820801 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.7049570083618164 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.705026149749756 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7050704956054688 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.705024003982544 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7049789428710938 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.70501971244812 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7049989700317383 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7049896717071533 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7051327228546143 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.704936981201172 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.70501971244812 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.7050328254699707 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.705090045928955 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.705015182495117 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.70491361618042 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.7049245834350586 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.7050509452819824 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.704986095428467 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.7049789428710938 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7049808502197266 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.705014228820801 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7050089836120605 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7049551010131836 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.7049460411071777 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.7050068378448486 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7050230503082275 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.704951524734497 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.704979419708252 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.704958200454712 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.704998016357422 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.7049331665039062 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.704957962036133 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7049927711486816 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.704991340637207 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7049927711486816 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7049736976623535 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.704949140548706 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.7049152851104736 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.7050070762634277 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.7050089836120605 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7050209045410156 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.704958915710449 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.7050249576568604 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.705029010772705 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7049171924591064 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.704970121383667 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.70504093170166 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.705014228820801 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.704927921295166 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.704878330230713 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.705003261566162 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7049763202667236 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7049901485443115 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.705026865005493 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7049753665924072 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.704988718032837 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7048890590667725 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.70495343208313 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.7050132751464844 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.705063819885254 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7050161361694336 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7050209045410156 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.705010175704956 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.705024003982544 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7049880027770996 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.704981803894043 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.705014944076538 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.7050063610076904 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.7049560546875 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.705031156539917 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.704935312271118 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7049190998077393 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7049319744110107 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.705008029937744 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7049763202667236 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.7049968242645264 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.7049691677093506 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.704954147338867 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.7048416137695312 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.704895257949829 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7049591541290283 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7050187587738037 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7049410343170166 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7049498558044434 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.704857110977173 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.7049272060394287 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.7051618099212646 + } + ] + }, + { + "bit": 15, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.701171636581421 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7011780738830566 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.701211452484131 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7011430263519287 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.701197624206543 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.7012646198272705 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.70119571685791 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.70113468170166 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.7011897563934326 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.7011220455169678 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7011232376098633 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.701225519180298 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.701108455657959 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.701124906539917 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7011618614196777 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.701227903366089 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.7010951042175293 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.701270818710327 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.701202630996704 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.701038360595703 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7012040615081787 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.701146364212036 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.7011916637420654 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.701153516769409 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.7011499404907227 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.701155185699463 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.7013423442840576 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.701201915740967 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7011122703552246 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.70125675201416 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.7012887001037598 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.701200008392334 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.7013092041015625 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.7011163234710693 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.7012224197387695 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7012248039245605 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.70119571685791 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.701092481613159 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.7011032104492188 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.701284170150757 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.701228618621826 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.701199531555176 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.7012457847595215 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7011847496032715 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7012577056884766 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.701164722442627 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7011148929595947 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.7010722160339355 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.7012319564819336 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.701157808303833 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.701173782348633 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.701253652572632 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7012288570404053 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.701087236404419 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7010679244995117 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7011241912841797 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.701150894165039 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.701237440109253 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.701145648956299 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7011451721191406 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.7011148929595947 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.701115131378174 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7011780738830566 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7011163234710693 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.7012035846710205 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.701162815093994 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.7010679244995117 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.7010767459869385 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.701235055923462 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.7012765407562256 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7011868953704834 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.701218843460083 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7010581493377686 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7011685371398926 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.701089382171631 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.7011570930480957 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7011923789978027 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.701200008392334 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.701171636581421 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7011847496032715 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7011897563934326 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.7012205123901367 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.701200008392334 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.701167583465576 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.701223611831665 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.701097249984741 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.701078414916992 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.701136827468872 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.701179027557373 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7011771202087402 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.7012388706207275 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7012155055999756 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.701239824295044 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.70123291015625 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7011239528656006 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7012155055999756 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7011890411376953 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.70121169090271 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.701173782348633 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.7010719776153564 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.701145887374878 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7012038230895996 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7012505531311035 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7011852264404297 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.701108455657959 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.701207399368286 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.701354503631592 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7012386322021484 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.7012226581573486 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.701183557510376 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7011361122131348 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.701162099838257 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.701094388961792 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7011945247650146 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.701145648956299 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.7011868953704834 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.7011446952819824 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.701179027557373 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.701331377029419 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.701195478439331 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.701199769973755 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7012643814086914 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7012388706207275 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7011051177978516 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.701179027557373 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7012081146240234 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.701078414916992 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7011220455169678 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.701202630996704 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.701215982437134 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.701181173324585 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.701209783554077 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.70125150680542 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.701258420944214 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7011725902557373 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.7012722492218018 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.701140880584717 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.7011685371398926 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.70120906829834 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.7013113498687744 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7011468410491943 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.701157569885254 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.7011966705322266 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7011702060699463 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.701174020767212 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.7011630535125732 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.7012524604797363 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7011067867279053 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.701228618621826 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.7011940479278564 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.701206922531128 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.7012317180633545 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.701143741607666 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.7012009620666504 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.701173782348633 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7011702060699463 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.701054573059082 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.701253652572632 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.70116925239563 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.701157569885254 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.701199531555176 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.7011728286743164 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.701141119003296 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7012057304382324 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7012205123901367 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.701200008392334 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7012248039245605 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.7012076377868652 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.7012317180633545 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.701185941696167 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.7011826038360596 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.701195478439331 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.701190710067749 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.7012691497802734 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.701132297515869 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.70109224319458 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.70114803314209 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7011799812316895 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7011940479278564 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7011852264404297 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.7011728286743164 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.701183557510376 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.701064109802246 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7012364864349365 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.7011489868164062 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7011852264404297 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7012248039245605 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7011561393737793 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.70108699798584 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.701105833053589 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7011518478393555 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.7011260986328125 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.701183795928955 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7012007236480713 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7012178897857666 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7011916637420654 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.7011940479278564 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.701152801513672 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.701153516769409 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.701174736022949 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.701199531555176 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.701133966445923 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7011477947235107 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.701068878173828 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.701159715652466 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.7011120319366455 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.70120906829834 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.701155185699463 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7011241912841797 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.70116925239563 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7011306285858154 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.7011570930480957 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7010951042175293 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.7011640071868896 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.701143264770508 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.701289415359497 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.7011518478393555 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.701136827468872 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.7011570930480957 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7012064456939697 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.701214551925659 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.7012245655059814 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.701215982437134 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.701122760772705 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.701129674911499 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.7012064456939697 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7011890411376953 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.701207399368286 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7011711597442627 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.701162099838257 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.7011759281158447 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.7011148929595947 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7012743949890137 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.70121169090271 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.7011919021606445 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7011606693267822 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7012240886688232 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7011044025421143 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.701179027557373 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.701154947280884 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7011377811431885 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7012038230895996 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7011826038360596 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7011566162109375 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.701202869415283 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.701183795928955 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.701167583465576 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7012367248535156 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.701146364212036 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.701246976852417 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.701263427734375 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.7012197971343994 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.70123028755188 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.7012546062469482 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.701202630996704 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7012267112731934 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.701174736022949 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.70120906829834 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.7011477947235107 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.701202869415283 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7012035846710205 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.701239824295044 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.7011945247650146 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.701235055923462 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.701242685317993 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7012457847595215 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.7011451721191406 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.701202869415283 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.7011313438415527 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.7012176513671875 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.701085090637207 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.701111078262329 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.701144218444824 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.701141119003296 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.701136827468872 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7012205123901367 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.701225757598877 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.70113468170166 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.701199531555176 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7010679244995117 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7011878490448 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7011899948120117 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.701215982437134 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.7011771202087402 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7011845111846924 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7012686729431152 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.701092481613159 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7012038230895996 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.701105833053589 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7011845111846924 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.701162099838257 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7011868953704834 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7011358737945557 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7011022567749023 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.7012417316436768 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7011680603027344 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7012691497802734 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7011561393737793 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.7011897563934326 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.701199769973755 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.7012135982513428 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7011680603027344 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.7011489868164062 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7012462615966797 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.7012205123901367 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7011280059814453 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.70123553276062 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.701253652572632 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.7012226581573486 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.701202630996704 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7011358737945557 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.70119571685791 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.701209545135498 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.7012200355529785 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7011733055114746 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.7012135982513428 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7012743949890137 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7011659145355225 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.701104164123535 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.701206922531128 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.701155185699463 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7012016773223877 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.7011218070983887 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.701221466064453 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7011711597442627 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7012007236480713 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7011706829071045 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7012665271759033 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.701133966445923 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7011032104492188 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7011587619781494 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.701171875 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.701164722442627 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.7012038230895996 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.701179027557373 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7011618614196777 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7011587619781494 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7012240886688232 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7012007236480713 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.701197624206543 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7011759281158447 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.7011520862579346 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.701223134994507 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.701173782348633 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7012295722961426 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7011756896972656 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.701143741607666 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.701155662536621 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7012147903442383 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.701104164123535 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.701155185699463 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.70105242729187 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.7011916637420654 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7011890411376953 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7011804580688477 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.701141595840454 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.7011280059814453 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7012698650360107 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.701141119003296 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7012245655059814 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7011749744415283 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.701162815093994 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.701136827468872 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.7012126445770264 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7012107372283936 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.701119899749756 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7011241912841797 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7012691497802734 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.701146364212036 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7012057304382324 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.701080322265625 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.701101064682007 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.701174736022949 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.701120376586914 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7011520862579346 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.701200008392334 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7010679244995117 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.701087236404419 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.7011659145355225 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.7011260986328125 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.7012057304382324 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.701164722442627 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.701158285140991 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.7011289596557617 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.701242685317993 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.701183557510376 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.701171875 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.701129674911499 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.701227903366089 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7011218070983887 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.701195001602173 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.701153516769409 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.7011122703552246 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7011301517486572 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7011919021606445 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7011702060699463 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.701190710067749 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.701174736022949 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.701199531555176 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.7011706829071045 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.7011380195617676 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.701228618621826 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.7011799812316895 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.7011306285858154 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.7011380195617676 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.7011845111846924 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.7011728286743164 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.701157808303833 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.701179027557373 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.701202869415283 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7012224197387695 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.701073169708252 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7011756896972656 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.701178789138794 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7011756896972656 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.7012438774108887 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7011542320251465 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.7011680603027344 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.701108932495117 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.7011659145355225 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.7012338638305664 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.7011868953704834 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7011876106262207 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.70119571685791 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.70119571685791 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.701190710067749 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.7011430263519287 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.7012763023376465 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.701150894165039 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7011878490448 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7011702060699463 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7011799812316895 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7012205123901367 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7011985778808594 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7011172771453857 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.701181650161743 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.701209783554077 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7011473178863525 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.701125144958496 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.701181650161743 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.701115131378174 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.7011733055114746 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.7011051177978516 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.701204776763916 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7011799812316895 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.701144218444824 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.7011868953704834 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7012436389923096 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.7011337280273438 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.7011570930480957 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.701145648956299 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.701167583465576 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.7011609077453613 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7011260986328125 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.7011866569519043 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.701159954071045 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.701097249984741 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.701180934906006 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7011446952819824 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.701119899749756 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7011377811431885 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.701129674911499 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.701089382171631 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.701190710067749 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7011349201202393 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.7011163234710693 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.7011032104492188 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.7010533809661865 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.701125144958496 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.70116925239563 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7011759281158447 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7011218070983887 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.701141119003296 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7011351585388184 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7011361122131348 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.701199531555176 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7011897563934326 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.701176643371582 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.701157808303833 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.701143264770508 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.701239585876465 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.701151132583618 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.7011187076568604 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.7012009620666504 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.701183557510376 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7011587619781494 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7011518478393555 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7010679244995117 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.701204776763916 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.701228618621826 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.701176643371582 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7011988162994385 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.7011077404022217 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7011468410491943 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.7011725902557373 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.7011916637420654 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.701200008392334 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7011609077453613 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7012226581573486 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.701132297515869 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7011845111846924 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7011659145355225 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7011330127716064 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.7012081146240234 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.7012007236480713 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.701174736022949 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.701183557510376 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.701167583465576 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.70109486579895 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.7011451721191406 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.701165199279785 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.701133966445923 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.7011280059814453 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.701120376586914 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.701202630996704 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7010793685913086 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7011401653289795 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7010891437530518 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.701218366622925 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7011725902557373 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7011866569519043 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7011876106262207 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.701097249984741 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7011420726776123 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.7011706829071045 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.7010581493377686 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.70121169090271 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7011706829071045 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.7011289596557617 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.701111078262329 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.701190710067749 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7012264728546143 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.7011570930480957 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.701166868209839 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.701192855834961 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.7011852264404297 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.701153516769409 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7012135982513428 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7011852264404297 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.701133966445923 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.7011706829071045 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.7011570930480957 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.701157808303833 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.70119309425354 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.7012717723846436 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7012453079223633 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.701157569885254 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7012081146240234 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.701174736022949 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7011630535125732 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7012267112731934 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.701111316680908 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.701078414916992 + } + ] + } + ] +} \ No newline at end of file