[3.2.10] NFSv3 CLOSE_WAIT hang

Hello!

I've reported this a few times now, but we still keep hitting the issue. NFS clients in the cluster occasionally get stuck with the socket to one of the NFS servers in CLOSE_WAIT state, and don't recover.

The NFS servers (knfsd) used to be running 2.6.30 since newer kernels hit XFS bugs, but are all now finally running 3.2.10 or 3.2.16.

I've added some debugging printk()s and WARN()s to the kernel to try to figure out what is going on. Without knowing much about how it's actually expected to work :), these were mostly to figure out why the socket is being closed. I added the following on top of with Trond's sunrpc debugging patch:

index 1e3b42c..c850c76 100644
@@ -1462,12 +1462,16 @@ call_status(struct rpc_task *task)
 		rpc_delay(task, 3*HZ);
 	case -ETIMEDOUT:
 		task->tk_action = call_timeout;
+		printk("%s: RPC call timeout (%d), disconnecting\n",
+			clnt->cl_protname, -status);
 		if (task->tk_client->cl_discrtry)
 			xprt_conditional_disconnect(task->tk_xprt,
 					req->rq_connect_cookie);
 		break;
 	case -ECONNRESET:
 	case -ECONNREFUSED:
+		printk("%s: RPC connection reset (%d), rebinding\n",
+			clnt->cl_protname, -status);
 		rpc_force_rebind(clnt);
 		rpc_delay(task, 3*HZ);
 	case -EPIPE:

index f1c57d4..9dc7870 100644
@@ -633,7 +633,12 @@ void xprt_force_disconnect(struct rpc_xprt *xprt)
 {
 	/* Don't race with the test_bit() in xprt_clear_locked() */
 	spin_lock_bh(&xprt->transport_lock);
+#if 0
 	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
+#else
+	if (test_and_set_bit(XPRT_CLOSE_WAIT, &xprt->state) == 0)
+		WARN(1, "xprt_force_disconnect(): setting XPRT_CLOSE_WAIT\n");
+#endif
 	/* Try to schedule an autoclose RPC call */
 	if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
 		BUG_ON(!queue_work(rpciod_workqueue, &xprt->task_cleanup));

@@ -660,7 +665,12 @@ void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
 		goto out;
 	if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
 		goto out;
+#if 0
 	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
+#else
+	if (test_and_set_bit(XPRT_CLOSE_WAIT, &xprt->state) == 0)
+		WARN(1, "xprt_conditional_disconnect(): setting XPRT_CLOSE_WAIT\n");
+#endif
 	/* Try to schedule an autoclose RPC call */
 	if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
 		BUG_ON(!queue_work(rpciod_workqueue, &xprt->task_cleanup));

@@ -706,8 +716,10 @@ void xprt_connect(struct rpc_task *task)
 	if (!xprt_lock_write(xprt, task))
 		return;
 
-	if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state))
+	if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
 		xprt->ops->close(xprt);
+		printk("RPC: xprt_connect: cleared XPRT_CLOSE_WAIT for xprt %p (pid %u)\n", xprt, task->tk_pid);
+	}
 
 	if (xprt_connected(xprt))
 		xprt_release_write(xprt, task);

index 55472c4..0ed3b99 100644
@@ -772,6 +772,8 @@ static void xs_tcp_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
 	if (req->rq_bytes_sent == req->rq_snd_buf.len)
 		goto out_release;
 	set_bit(XPRT_CLOSE_WAIT, &task->tk_xprt->state);
+	WARN(1, "xs_tcp_release_xprt(): setting XPRT_CLOSE_WAIT -- req->rq_bytes_sent=%u, req->rq_snd_buf.len=%u\n",
+		req->rq_bytes_sent, req->rq_snd_buf.len);
 out_release:
 	xprt_release_xprt(xprt, task);
 }

@@ -1517,6 +1519,11 @@ static void xs_tcp_state_change(struct sock *sk)
 		/* The server initiated a shutdown of the socket */
 		xprt_force_disconnect(xprt);
 		xprt->connect_cookie++;
+		printk("RPC: server initated shutdown -- state %x conn %d dead %d zapped %d sk_shutdown %d\n",
+			sk->sk_state, xprt_connected(xprt),
+			sock_flag(sk, SOCK_DEAD),
+			sock_flag(sk, SOCK_ZAPPED),
+			sk->sk_shutdown);
 	case TCP_CLOSING:
 		/*
 		 * If the server closed down the connection, make sure that

---

Mount options on the clients: rw,hard,intr,tcp,timeo=300,retrans=2,vers=3 ...the timeout is primarily left over from an attempt to make HA fail-over "reconnection" times faster, since with UDP it actually did make a significant difference. Does this even matter with TCP sockets?

Here, we hit the CLOSE_WAIT state. There are a number of occasional "RPC call timeout (110), disconnecting" in normal operation; I'm not sure why this is.

[1119196.640017] nfs: RPC call timeout (110), disconnecting
[1119196.720026] nfs: RPC call timeout (110), disconnecting
[1119196.892021] nfs: RPC call timeout (110), disconnecting
[1119198.516022] nfs: RPC call timeout (110), disconnecting
[1119220.576025] nfs: RPC call timeout (110), disconnecting
[1119871.048040] nfs: RPC call timeout (110), disconnecting
[1120266.464021] nfs: RPC call timeout (110), disconnecting
[1120341.472021] nfs: RPC call timeout (110), disconnecting
[1122139.008022] nfs: RPC call timeout (110), disconnecting
[1122166.112043] nfs: RPC call timeout (110), disconnecting
[1122302.304037] nfs: RPC call timeout (110), disconnecting
[1122666.735765] ------------[ cut here ]------------
[1122666.735855] WARNING: at net/sunrpc/xprt.c:640 xprt_force_disconnect+0x4a/0xa0()
[1122666.735945] Hardware name: PowerEdge 1950
[1122666.736025] xprt_force_disconnect(): setting XPRT_CLOSE_WAIT
[1122666.736099] Modules linked in: ipmi_devintf ipmi_si ipmi_msghandler xt_recent nf_conntrack_ftp xt_state xt_owner nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 bnx2 serio_raw evdev
[1122666.736898] Pid: 18458, comm: httpd Tainted: G        W    3.2.10-hw+ #16
[1122666.736975] Call Trace:
[1122666.737043]  <IRQ>  [<ffffffff816b545a>] ? xprt_force_disconnect+0x4a/0xa0
[1122666.737176]  [<ffffffff8105dd60>] warn_slowpath_common+0x80/0xc0
[1122666.737252]  [<ffffffff8105de99>] warn_slowpath_fmt+0x69/0x70
[1122666.737335]  [<ffffffffa0055aff>] ? tcp_packet+0x7cf/0x10f0 [nf_conntrack]
[1122666.737414]  [<ffffffff81702c41>] ? _raw_spin_lock_bh+0x11/0x40
[1122666.737490]  [<ffffffff816b545a>] xprt_force_disconnect+0x4a/0xa0
[1122666.737566]  [<ffffffff816b87e8>] xs_tcp_state_change+0x78/0x320
[1122666.737643]  [<ffffffff81667f4b>] tcp_fin+0x17b/0x210
[1122666.737717]  [<ffffffff81668c52>] tcp_data_queue+0x4c2/0xe70
[1122666.737793]  [<ffffffff8166c389>] ? tcp_validate_incoming+0x109/0x340
[1122666.737869]  [<ffffffff8166c709>] tcp_rcv_established+0x149/0xb00
[1122666.737946]  [<ffffffff816751dc>] tcp_v4_do_rcv+0x15c/0x2e0
[1122666.738021]  [<ffffffff81649924>] ? nf_iterate+0x84/0xb0
[1122666.738096]  [<ffffffff816759af>] tcp_v4_rcv+0x64f/0x8b0
[1122666.738170]  [<ffffffff81649a9d>] ? nf_hook_slow+0x6d/0x130
[1122666.738246]  [<ffffffff816543b0>] ? ip_rcv+0x2f0/0x2f0
[1122666.738322]  [<ffffffff8165445a>] ip_local_deliver_finish+0xaa/0x1d0
[1122666.738399]  [<ffffffff8165460d>] ip_local_deliver+0x8d/0xa0
[1122666.738474]  [<ffffffff81653ef9>] ip_rcv_finish+0x169/0x330
[1122666.738549]  [<ffffffff8165433d>] ip_rcv+0x27d/0x2f0
[1122666.738625]  [<ffffffff816256e7>] __netif_receive_skb+0x437/0x490
[1122666.738701]  [<ffffffff81627568>] netif_receive_skb+0x78/0x80
[1122666.738778]  [<ffffffff813aa8dc>] ? is_swiotlb_buffer+0x3c/0x50
[1122666.738854]  [<ffffffff816276a0>] napi_skb_finish+0x50/0x70
[1122666.738929]  [<ffffffff81627ce5>] napi_gro_receive+0x105/0x140
[1122666.739008]  [<ffffffffa0029e7b>] bnx2_poll_work+0x60b/0x1550 [bnx2]
[1122666.739085]  [<ffffffff8105096d>] ? enqueue_task_fair+0x15d/0x540
[1122666.739163]  [<ffffffff8104825e>] ? cpu_smt_mask+0x1e/0x20
[1122666.739238]  [<ffffffff810475dd>] ? check_preempt_curr+0x7d/0xa0
[1122666.739315]  [<ffffffffa002aeec>] bnx2_poll+0x5c/0x230 [bnx2]
[1122666.739391]  [<ffffffff81627f0b>] net_rx_action+0x11b/0x2e0
[1122666.739467]  [<ffffffff81019276>] ? read_tsc+0x16/0x40
[1122666.739542]  [<ffffffff81064fa0>] __do_softirq+0xe0/0x1d0
[1122666.739619]  [<ffffffff8170c8ec>] call_softirq+0x1c/0x30
[1122666.739693]  [<ffffffff810142b5>] do_softirq+0x65/0xa0
[1122666.739767]  [<ffffffff81064cad>] irq_exit+0xad/0xe0
[1122666.739841]  [<ffffffff81013a94>] do_IRQ+0x64/0xe0
[1122666.739914]  [<ffffffff81702dee>] common_interrupt+0x6e/0x6e
[1122666.739988]  <EOI>  [<ffffffff8170a712>] ? system_call_fastpath+0x16/0x1b
[1122666.740127] ---[ end trace 18327738ff1711ee ]---
[1122666.740203] RPC: server initated shutdown -- state 8 conn 1 dead 0 zapped 1 sk_shutdown 1

No other messages after this -- we see the socket in CLOSE_WAIT:

Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name Timer
tcp        0      0 10.10.52.11:801         10.10.52.225:2049       CLOSE_WAIT  -                off (0.00/0/0)

Here's the dmesg after "rpcdebug -m rpc -c all": http://0x.ca/sim/ref/3.2.10/dmesg

Where should I go from here? Is it valid that some processes should get stuck in uninterruptible sleep in this case? Is it valid that the server should hang up at all?

Cheers,

Simon-

T24gVHVlLCAyMDEyLTA0LTI0IGF0IDE3OjU2IC0wNzAwLCBTaW1vbiBLaXJieSB3cm90ZToNCj4g SGVsbG8hDQo+IA0KPiBJJ3ZlIHJlcG9ydGVkIHRoaXMgYSBmZXcgdGltZXMgbm93LCBidXQgd2Ug c3RpbGwga2VlcCBoaXR0aW5nIHRoZSBpc3N1ZS4NCj4gTkZTIGNsaWVudHMgaW4gdGhlIGNsdXN0 ZXIgb2NjYXNpb25hbGx5IGdldCBzdHVjayB3aXRoIHRoZSBzb2NrZXQgdG8gb25lDQo+IG9mIHRo ZSBORlMgc2VydmVycyBpbiBDTE9TRV9XQUlUIHN0YXRlLCBhbmQgZG9uJ3QgcmVjb3Zlci4NCj4g DQo+IFRoZSBORlMgc2VydmVycyAoa25mc2QpIHVzZWQgdG8gYmUgcnVubmluZyAyLjYuMzAgc2lu Y2UgbmV3ZXIga2VybmVscyBoaXQNCj4gWEZTIGJ1Z3MsIGJ1dCBhcmUgYWxsIG5vdyBmaW5hbGx5 IHJ1bm5pbmcgMy4yLjEwIG9yIDMuMi4xNi4NCj4gDQo+IEkndmUgYWRkZWQgc29tZSBkZWJ1Z2dp bmcgcHJpbnRrKClzIGFuZCBXQVJOKClzIHRvIHRoZSBrZXJuZWwgdG8gdHJ5IHRvDQo+IGZpZ3Vy ZSBvdXQgd2hhdCBpcyBnb2luZyBvbi4gV2l0aG91dCBrbm93aW5nIG11Y2ggYWJvdXQgaG93IGl0 J3MgYWN0dWFsbHkNCj4gZXhwZWN0ZWQgdG8gd29yayA6KSwgdGhlc2Ugd2VyZSBtb3N0bHkgdG8g ZmlndXJlIG91dCB3aHkgdGhlIHNvY2tldCBpcw0KPiBiZWluZyBjbG9zZWQuIEkgYWRkZWQgdGhl IGZvbGxvd2luZyBvbiB0b3Agb2Ygd2l0aCBUcm9uZCdzIHN1bnJwYw0KPiBkZWJ1Z2dpbmcgcGF0 Y2g6DQo8c25pcD4NCj4gWzExMTkxOTYuNjQwMDE3XSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDEx MCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkxOTYuNzIwMDI2XSBuZnM6IFJQQyBjYWxsIHRpbWVv dXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkxOTYuODkyMDIxXSBuZnM6IFJQQyBjYWxs IHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkxOTguNTE2MDIyXSBuZnM6IFJQ QyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTkyMjAuNTc2MDI1XSBu ZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMTk4NzEuMDQ4 MDQwXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMjAy NjYuNDY0MDIxXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3RpbmcNCj4g WzExMjAzNDEuNDcyMDIxXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nvbm5lY3Rp bmcNCj4gWzExMjIxMzkuMDA4MDIyXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCksIGRpc2Nv bm5lY3RpbmcNCj4gWzExMjIxNjYuMTEyMDQzXSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQgKDExMCks IGRpc2Nvbm5lY3RpbmcNCj4gWzExMjIzMDIuMzA0MDM3XSBuZnM6IFJQQyBjYWxsIHRpbWVvdXQg KDExMCksIGRpc2Nvbm5lY3RpbmcNCj4gWzExMjI2NjYuNzM1NzY1XSAtLS0tLS0tLS0tLS1bIGN1 dCBoZXJlIF0tLS0tLS0tLS0tLS0NCj4gWzExMjI2NjYuNzM1ODU1XSBXQVJOSU5HOiBhdCBuZXQv c3VucnBjL3hwcnQuYzo2NDAgeHBydF9mb3JjZV9kaXNjb25uZWN0KzB4NGEvMHhhMCgpDQo+IFsx MTIyNjY2LjczNTk0NV0gSGFyZHdhcmUgbmFtZTogUG93ZXJFZGdlIDE5NTANCj4gWzExMjI2NjYu NzM2MDI1XSB4cHJ0X2ZvcmNlX2Rpc2Nvbm5lY3QoKTogc2V0dGluZyBYUFJUX0NMT1NFX1dBSVQN Cj4gWzExMjI2NjYuNzM2MDk5XSBNb2R1bGVzIGxpbmtlZCBpbjogaXBtaV9kZXZpbnRmIGlwbWlf c2kgaXBtaV9tc2doYW5kbGVyIHh0X3JlY2VudCBuZl9jb25udHJhY2tfZnRwIHh0X3N0YXRlIHh0 X293bmVyIG5mX2Nvbm50cmFja19pcHY0IG5mX2Nvbm50cmFjayBuZl9kZWZyYWdfaXB2NCBibngy IHNlcmlvX3JhdyBldmRldg0KPiBbMTEyMjY2Ni43MzY4OThdIFBpZDogMTg0NTgsIGNvbW06IGh0 dHBkIFRhaW50ZWQ6IEcgICAgICAgIFcgICAgMy4yLjEwLWh3KyAjMTYNCj4gWzExMjI2NjYuNzM2 OTc1XSBDYWxsIFRyYWNlOg0KPiBbMTEyMjY2Ni43MzcwNDNdICA8SVJRPiAgWzxmZmZmZmZmZjgx NmI1NDVhPl0gPyB4cHJ0X2ZvcmNlX2Rpc2Nvbm5lY3QrMHg0YS8weGEwDQo+IFsxMTIyNjY2Ljcz NzE3Nl0gIFs8ZmZmZmZmZmY4MTA1ZGQ2MD5dIHdhcm5fc2xvd3BhdGhfY29tbW9uKzB4ODAvMHhj MA0KPiBbMTEyMjY2Ni43MzcyNTJdICBbPGZmZmZmZmZmODEwNWRlOTk+XSB3YXJuX3Nsb3dwYXRo X2ZtdCsweDY5LzB4NzANCj4gWzExMjI2NjYuNzM3MzM1XSAgWzxmZmZmZmZmZmEwMDU1YWZmPl0g PyB0Y3BfcGFja2V0KzB4N2NmLzB4MTBmMCBbbmZfY29ubnRyYWNrXQ0KPiBbMTEyMjY2Ni43Mzc0 MTRdICBbPGZmZmZmZmZmODE3MDJjNDE+XSA/IF9yYXdfc3Bpbl9sb2NrX2JoKzB4MTEvMHg0MA0K PiBbMTEyMjY2Ni43Mzc0OTBdICBbPGZmZmZmZmZmODE2YjU0NWE+XSB4cHJ0X2ZvcmNlX2Rpc2Nv bm5lY3QrMHg0YS8weGEwDQo+IFsxMTIyNjY2LjczNzU2Nl0gIFs8ZmZmZmZmZmY4MTZiODdlOD5d IHhzX3RjcF9zdGF0ZV9jaGFuZ2UrMHg3OC8weDMyMA0KPiBbMTEyMjY2Ni43Mzc2NDNdICBbPGZm ZmZmZmZmODE2NjdmNGI+XSB0Y3BfZmluKzB4MTdiLzB4MjEwDQo+IFsxMTIyNjY2LjczNzcxN10g IFs8ZmZmZmZmZmY4MTY2OGM1Mj5dIHRjcF9kYXRhX3F1ZXVlKzB4NGMyLzB4ZTcwDQo+IFsxMTIy NjY2LjczNzc5M10gIFs8ZmZmZmZmZmY4MTY2YzM4OT5dID8gdGNwX3ZhbGlkYXRlX2luY29taW5n KzB4MTA5LzB4MzQwDQo+IFsxMTIyNjY2LjczNzg2OV0gIFs8ZmZmZmZmZmY4MTY2YzcwOT5dIHRj cF9yY3ZfZXN0YWJsaXNoZWQrMHgxNDkvMHhiMDANCj4gWzExMjI2NjYuNzM3OTQ2XSAgWzxmZmZm ZmZmZjgxNjc1MWRjPl0gdGNwX3Y0X2RvX3JjdisweDE1Yy8weDJlMA0KPiBbMTEyMjY2Ni43Mzgw MjFdICBbPGZmZmZmZmZmODE2NDk5MjQ+XSA/IG5mX2l0ZXJhdGUrMHg4NC8weGIwDQo+IFsxMTIy NjY2LjczODA5Nl0gIFs8ZmZmZmZmZmY4MTY3NTlhZj5dIHRjcF92NF9yY3YrMHg2NGYvMHg4YjAN Cj4gWzExMjI2NjYuNzM4MTcwXSAgWzxmZmZmZmZmZjgxNjQ5YTlkPl0gPyBuZl9ob29rX3Nsb3cr MHg2ZC8weDEzMA0KPiBbMTEyMjY2Ni43MzgyNDZdICBbPGZmZmZmZmZmODE2NTQzYjA+XSA/IGlw X3JjdisweDJmMC8weDJmMA0KPiBbMTEyMjY2Ni43MzgzMjJdICBbPGZmZmZmZmZmODE2NTQ0NWE+ XSBpcF9sb2NhbF9kZWxpdmVyX2ZpbmlzaCsweGFhLzB4MWQwDQo+IFsxMTIyNjY2LjczODM5OV0g IFs8ZmZmZmZmZmY4MTY1NDYwZD5dIGlwX2xvY2FsX2RlbGl2ZXIrMHg4ZC8weGEwDQo+IFsxMTIy NjY2LjczODQ3NF0gIFs8ZmZmZmZmZmY4MTY1M2VmOT5dIGlwX3Jjdl9maW5pc2grMHgxNjkvMHgz MzANCj4gWzExMjI2NjYuNzM4NTQ5XSAgWzxmZmZmZmZmZjgxNjU0MzNkPl0gaXBfcmN2KzB4Mjdk LzB4MmYwDQo+IFsxMTIyNjY2LjczODYyNV0gIFs8ZmZmZmZmZmY4MTYyNTZlNz5dIF9fbmV0aWZf cmVjZWl2ZV9za2IrMHg0MzcvMHg0OTANCj4gWzExMjI2NjYuNzM4NzAxXSAgWzxmZmZmZmZmZjgx NjI3NTY4Pl0gbmV0aWZfcmVjZWl2ZV9za2IrMHg3OC8weDgwDQo+IFsxMTIyNjY2LjczODc3OF0g IFs8ZmZmZmZmZmY4MTNhYThkYz5dID8gaXNfc3dpb3RsYl9idWZmZXIrMHgzYy8weDUwDQo+IFsx MTIyNjY2LjczODg1NF0gIFs8ZmZmZmZmZmY4MTYyNzZhMD5dIG5hcGlfc2tiX2ZpbmlzaCsweDUw LzB4NzANCj4gWzExMjI2NjYuNzM4OTI5XSAgWzxmZmZmZmZmZjgxNjI3Y2U1Pl0gbmFwaV9ncm9f cmVjZWl2ZSsweDEwNS8weDE0MA0KPiBbMTEyMjY2Ni43MzkwMDhdICBbPGZmZmZmZmZmYTAwMjll N2I+XSBibngyX3BvbGxfd29yaysweDYwYi8weDE1NTAgW2JueDJdDQo+IFsxMTIyNjY2LjczOTA4 NV0gIFs8ZmZmZmZmZmY4MTA1MDk2ZD5dID8gZW5xdWV1ZV90YXNrX2ZhaXIrMHgxNWQvMHg1NDAN Cj4gWzExMjI2NjYuNzM5MTYzXSAgWzxmZmZmZmZmZjgxMDQ4MjVlPl0gPyBjcHVfc210X21hc2sr MHgxZS8weDIwDQo+IFsxMTIyNjY2LjczOTIzOF0gIFs8ZmZmZmZmZmY4MTA0NzVkZD5dID8gY2hl Y2tfcHJlZW1wdF9jdXJyKzB4N2QvMHhhMA0KPiBbMTEyMjY2Ni43MzkzMTVdICBbPGZmZmZmZmZm YTAwMmFlZWM+XSBibngyX3BvbGwrMHg1Yy8weDIzMCBbYm54Ml0NCj4gWzExMjI2NjYuNzM5Mzkx XSAgWzxmZmZmZmZmZjgxNjI3ZjBiPl0gbmV0X3J4X2FjdGlvbisweDExYi8weDJlMA0KPiBbMTEy MjY2Ni43Mzk0NjddICBbPGZmZmZmZmZmODEwMTkyNzY+XSA/IHJlYWRfdHNjKzB4MTYvMHg0MA0K PiBbMTEyMjY2Ni43Mzk1NDJdICBbPGZmZmZmZmZmODEwNjRmYTA+XSBfX2RvX3NvZnRpcnErMHhl MC8weDFkMA0KPiBbMTEyMjY2Ni43Mzk2MTldICBbPGZmZmZmZmZmODE3MGM4ZWM+XSBjYWxsX3Nv ZnRpcnErMHgxYy8weDMwDQo+IFsxMTIyNjY2LjczOTY5M10gIFs8ZmZmZmZmZmY4MTAxNDJiNT5d IGRvX3NvZnRpcnErMHg2NS8weGEwDQo+IFsxMTIyNjY2LjczOTc2N10gIFs8ZmZmZmZmZmY4MTA2 NGNhZD5dIGlycV9leGl0KzB4YWQvMHhlMA0KPiBbMTEyMjY2Ni43Mzk4NDFdICBbPGZmZmZmZmZm ODEwMTNhOTQ+XSBkb19JUlErMHg2NC8weGUwDQo+IFsxMTIyNjY2LjczOTkxNF0gIFs8ZmZmZmZm ZmY4MTcwMmRlZT5dIGNvbW1vbl9pbnRlcnJ1cHQrMHg2ZS8weDZlDQo+IFsxMTIyNjY2LjczOTk4 OF0gIDxFT0k+ICBbPGZmZmZmZmZmODE3MGE3MTI+XSA/IHN5c3RlbV9jYWxsX2Zhc3RwYXRoKzB4 MTYvMHgxYg0KPiBbMTEyMjY2Ni43NDAxMjddIC0tLVsgZW5kIHRyYWNlIDE4MzI3NzM4ZmYxNzEx ZWUgXS0tLQ0KPiBbMTEyMjY2Ni43NDAyMDNdIFJQQzogc2VydmVyIGluaXRhdGVkIHNodXRkb3du IC0tIHN0YXRlIDggY29ubiAxIGRlYWQgMCB6YXBwZWQgMSBza19zaHV0ZG93biAxDQo+IA0KPiBO byBvdGhlciBtZXNzYWdlcyBhZnRlciB0aGlzIC0tIHdlIHNlZSB0aGUgc29ja2V0IGluIENMT1NF X1dBSVQ6DQo+IA0KPiBQcm90byBSZWN2LVEgU2VuZC1RIExvY2FsIEFkZHJlc3MgICAgICAgICAg IEZvcmVpZ24gQWRkcmVzcyAgICAgICAgIFN0YXRlICAgICAgIFBJRC9Qcm9ncmFtIG5hbWUgVGlt ZXINCj4gdGNwICAgICAgICAwICAgICAgMCAxMC4xMC41Mi4xMTo4MDEgICAgICAgICAxMC4xMC41 Mi4yMjU6MjA0OSAgICAgICBDTE9TRV9XQUlUICAtICAgICAgICAgICAgICAgIG9mZiAoMC4wMC8w LzApDQo+IA0KPiBIZXJlJ3MgdGhlIGRtZXNnIGFmdGVyICJycGNkZWJ1ZyAtbSBycGMgLWMgYWxs IjogaHR0cDovLzB4LmNhL3NpbS9yZWYvMy4yLjEwL2RtZXNnDQo+IA0KPiBXaGVyZSBzaG91bGQg SSBnbyBmcm9tIGhlcmU/IElzIGl0IHZhbGlkIHRoYXQgc29tZSBwcm9jZXNzZXMgc2hvdWxkIGdl dA0KPiBzdHVjayBpbiB1bmludGVycnVwdGlibGUgc2xlZXAgaW4gdGhpcyBjYXNlPyBJcyBpdCB2 YWxpZCB0aGF0IHRoZSBzZXJ2ZXINCj4gc2hvdWxkIGhhbmcgdXAgYXQgYWxsPw0KDQpDYW4geW91 IHRyeSBhcHBseWluZyBzb21ldGhpbmcgbGlrZSB0aGUgZm9sbG93aW5nIHBhdGNoIG9uIHRvcCBv ZiB3aGF0DQp5b3UgaGF2ZSBhcHBsaWVkIGFib3ZlPyBJdCBpcyBjbGVhciB0aGF0IHNvbWV0aGlu ZyBpcyBnb2luZyB3cm9uZyBpbiB0aGUNCmF1dG9kaXNjb25uZWN0IGxvZ2ljLCBidXQgaXQgaXMg bm90IHlldCAxMDAlIGNsZWFyIHRvIG1lIHdoYXQgdGhhdCBtaWdodA0KYmUuIFRoZSBwYXRjaCBi ZWxvdyBjaGVja3MgZm9yIHJhY2VzIGluIHhwcnRfYXV0b2Nsb3NlKCkgYW5kIGFsc28gY2hlY2tz DQp0byBzZWUgaWYgdGhlIHhwcnRfZm9yY2VfZGlzY29ubmVjdCBtaWdodCBiZSBmYWlsaW5nIHRv IHdha2UgdXAgYSBzdHVjaw0KdGFzayB0aGF0IGhvbGRzIHRoZSBzb2NrZXQgd3JpdGUgbG9jay4N Cg0KQ2hlZXJzDQogIFRyb25kDQotLS0NCmRpZmYgLS1naXQgYS9uZXQvc3VucnBjL3hwcnQuYyBi L25ldC9zdW5ycGMveHBydC5jDQppbmRleCBjNjRjMGVmLi4xOTA0NDY3IDEwMDY0NA0KLS0tIGEv bmV0L3N1bnJwYy94cHJ0LmMNCisrKyBiL25ldC9zdW5ycGMveHBydC5jDQpAQCAtNjA0LDggKzYw NCw5IEBAIHN0YXRpYyB2b2lkIHhwcnRfYXV0b2Nsb3NlKHN0cnVjdCB3b3JrX3N0cnVjdCAqd29y aykNCiAJc3RydWN0IHJwY194cHJ0ICp4cHJ0ID0NCiAJCWNvbnRhaW5lcl9vZih3b3JrLCBzdHJ1 Y3QgcnBjX3hwcnQsIHRhc2tfY2xlYW51cCk7DQogDQotCXhwcnQtPm9wcy0+Y2xvc2UoeHBydCk7 DQogCWNsZWFyX2JpdChYUFJUX0NMT1NFX1dBSVQsICZ4cHJ0LT5zdGF0ZSk7DQorCXhwcnQtPm9w cy0+Y2xvc2UoeHBydCk7DQorCVdBUk5fT04odGVzdF9iaXQoWFBSVF9DTE9TRV9XQUlULCAmeHBy dC0+c3RhdGUpKTsNCiAJeHBydF9yZWxlYXNlX3dyaXRlKHhwcnQsIE5VTEwpOw0KIH0NCiANCkBA IC02MzgsNiArNjM5LDE0IEBAIHZvaWQgeHBydF9mb3JjZV9kaXNjb25uZWN0KHN0cnVjdCBycGNf eHBydCAqeHBydCkNCiAJaWYgKHRlc3RfYW5kX3NldF9iaXQoWFBSVF9MT0NLRUQsICZ4cHJ0LT5z dGF0ZSkgPT0gMCkNCiAJCXF1ZXVlX3dvcmsocnBjaW9kX3dvcmtxdWV1ZSwgJnhwcnQtPnRhc2tf Y2xlYW51cCk7DQogCXhwcnRfd2FrZV9wZW5kaW5nX3Rhc2tzKHhwcnQsIC1FQUdBSU4pOw0KKwlp ZiAoeHBydC0+c25kX3Rhc2sgIT0gTlVMTCkgew0KKwkJc3RydWN0IHJwY190YXNrICp0YXNrID0g eHBydC0+c25kX3Rhc2s7DQorCQlpZiAoUlBDX0lTX1FVRVVFRCh0YXNrKSkgew0KKwkJCVdBUk4o MSwgInRhc2sgJTV1IGlzIHF1ZXVlZCBvbiAlc1xuIiwNCisJCQkJCXRhc2stPnRrX3BpZCwNCisJ CQkJCXJwY19xbmFtZSh0YXNrLT50a193YWl0cXVldWUpKTsNCisJCX0NCisJfQ0KIAlzcGluX3Vu bG9ja19iaCgmeHBydC0+dHJhbnNwb3J0X2xvY2spOw0KIH0NCiANCg0KLS0gDQpUcm9uZCBNeWts ZWJ1c3QNCkxpbnV4IE5GUyBjbGllbnQgbWFpbnRhaW5lcg0KDQpOZXRBcHANClRyb25kLk15a2xl YnVzdEBuZXRhcHAuY29tDQp3d3cubmV0YXBwLmNvbQ0KDQo=