Add hint message for MTU settings when IC reports ERROR "Failed to send packet" (#17164)

* Add hint message for MTU settings when IC reports ERROR "Failed to send packet".

Author: z-wenlin

contrib/interconnect/udp/ic_faultinjection.h

@@ -83,6 +83,7 @@ typedef enum {
 	FINC_OS_NET_INTERFACE = 19,
 	FINC_OS_MEM_INTERFACE = 20,
 	FINC_OS_CREATE_THREAD = 21,
+	FINC_PKT_TOO_LONG = 22,
 
 	/* These are used to inject network faults. */
 	FINC_NET_RECV_ERROR = 23,
@@ -301,6 +302,13 @@ testmode_sendto(const char *caller_name, int socket, const void *buffer,
 			errno = EFAULT;
 			return -1;
 
+		case FINC_PKT_TOO_LONG:
+			if (!FINC_HAS_FAULT(fault_type) || !is_pkt)
+				break;
+			write_log("inject fault to sendto: FINC_PKT_TOO_LONG");
+			errno = EMSGSIZE;
+			return -1;
+
 		default:
 			break;
 	}

contrib/interconnect/udp/ic_udpifc.c

@@ -2246,10 +2246,23 @@ sendControlMessage(icpkthdr *pkt, int fd, struct sockaddr *addr, socklen_t peerL
 	if (gp_interconnect_full_crc)
 		addCRC(pkt);
 
-	char errDetail[100];
-	snprintf(errDetail, sizeof(errDetail), "Send control message: got error with seq %u", pkt->seq);
-	/* Retry for infinite times since we have no retransmit mechanism for control message */
-	n = sendtoWithRetry(fd, (const char *) pkt, pkt->len, 0, addr, peerLen, -1, errDetail);
+	/* retry 10 times for sending control message */
+	int counter = 0;
+	while (counter < 10)
+	{
+		counter++;
+		n = sendto(fd, (const char *) pkt, pkt->len, 0, addr, peerLen);
+		if (n < 0)
+		{
+			if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
+				continue;
+			else {
+				write_log("sendcontrolmessage: got errno %d", errno);
+				return;
+			}
+		}
+		break;
+	}
 	if (n < pkt->len)
 		write_log("sendcontrolmessage: got error %d errno %d seq %d", n, errno, pkt->seq);
 }
@@ -5392,6 +5405,19 @@ sendtoWithRetry(int socket, const void *message, size_t length,
 			return n;
 		}
 
+		/*
+		 * If the OS can detect an MTU issue on the host network interfaces, we
+		 * would get EMSGSIZE here. So, bail with a HINT about checking MTU.
+		 */
+		if (errno == EMSGSIZE)
+		{
+			ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
+							errmsg("Interconnect error writing an outgoing packet: %m"),
+							errdetail("error during sendto() call (error:%d).\n"
+									"%s", save_errno, errDetail),
+							errhint("check if interface MTU is equal across the cluster and lower than gp_max_packet_size")));
+		}
+
 		ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),
 						errmsg("Interconnect error writing an outgoing packet: %m"),
 						errdetail("error during sendto() call (error:%d).\n"

src/test/regress/expected/icudp/icudp_full.out

@@ -544,6 +544,14 @@ SELECT system_call_fault_injection_test();
 
 (1 row)
 
+-- inject faults for errMsgSize when packet is too long.
+SET gp_udpic_fault_inject_bitmap = 4194304;
+SELECT system_call_fault_injection_test();
+ system_call_fault_injection_test
+----------------------------------
+ 
+(1 row)
+
 -- disable ipv6 may increase the code coverage.
 SET gp_udpic_network_disable_ipv6 = 1;
 SELECT system_call_fault_injection_test();

src/test/regress/sql/icudp/icudp_full.sql

@@ -276,6 +276,10 @@ $$;
 SET gp_udpic_fault_inject_bitmap = 524288;
 SELECT system_call_fault_injection_test();
 
+-- inject faults for errMsgSize when packet is too long.
+SET gp_udpic_fault_inject_bitmap = 4194304;
+SELECT system_call_fault_injection_test();
+
 -- disable ipv6 may increase the code coverage.
 SET gp_udpic_network_disable_ipv6 = 1;
 SELECT system_call_fault_injection_test();