--- old/usr/src/uts/common/io/mac/mac_client.c	Wed Jan 22 11:04:53 2020
+++ new/usr/src/uts/common/io/mac/mac_client.c	Wed Jan 22 11:04:53 2020
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
+ * Copyright 2018 Joyent, Inc.
  * Copyright 2017 RackTop Systems.
  */
 
@@ -865,9 +865,12 @@
 	mac_protect_update_mac_token(mcip);
 
 	/*
-	 * A MAC client could have one MAC address but multiple
-	 * VLANs. In that case update the flow entries corresponding
-	 * to all VLANs of the MAC client.
+	 * When there are multiple VLANs sharing the same MAC address,
+	 * each gets its own MAC client, except when running on sun4v
+	 * vsw. In that case the mci_flent_list is used to place
+	 * multiple VLAN flows on one MAC client. If we ever get rid
+	 * of vsw then this code can go, but until then we need to
+	 * update all flow entries.
 	 */
 	for (flent = mcip->mci_flent_list; flent != NULL;
 	    flent = flent->fe_client_next) {
@@ -1025,7 +1028,7 @@
 		return (0);
 	}
 
-	if (mac_find_macaddr(mip, (uint8_t *)addr) != 0) {
+	if (mac_find_macaddr(mip, (uint8_t *)addr) != NULL) {
 		i_mac_perim_exit(mip);
 		return (EBUSY);
 	}
@@ -1040,9 +1043,9 @@
 		mac_capab_aggr_t aggr_cap;
 
 		/*
-		 * If the mac is an aggregation, other than the unicast
+		 * If the MAC is an aggregation, other than the unicast
 		 * addresses programming, aggr must be informed about this
-		 * primary unicst address change to change its mac address
+		 * primary unicst address change to change its MAC address
 		 * policy to be user-specified.
 		 */
 		ASSERT(map->ma_type == MAC_ADDRESS_TYPE_UNICAST_CLASSIFIED);
@@ -1374,7 +1377,7 @@
 		mcip->mci_state_flags |= MCIS_IS_AGGR_PORT;
 
 	if (mip->mi_state_flags & MIS_IS_AGGR)
-		mcip->mci_state_flags |= MCIS_IS_AGGR;
+		mcip->mci_state_flags |= MCIS_IS_AGGR_CLIENT;
 
 	if ((flags & MAC_OPEN_FLAGS_USE_DATALINK_NAME) != 0) {
 		datalink_id_t	linkid;
@@ -1539,7 +1542,8 @@
 }
 
 /*
- * Set the rx bypass receive callback.
+ * Set the Rx bypass receive callback and return B_TRUE. Return
+ * B_FALSE if it's not possible to enable bypass.
  */
 boolean_t
 mac_rx_bypass_set(mac_client_handle_t mch, mac_direct_rx_t rx_fn, void *arg1)
@@ -1550,11 +1554,11 @@
 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
 
 	/*
-	 * If the mac_client is a VLAN, we should not do DLS bypass and
-	 * instead let the packets come up via mac_rx_deliver so the vlan
-	 * header can be stripped.
+	 * If the client has more than one VLAN then process packets
+	 * through DLS. This should happen only when sun4v vsw is on
+	 * the scene.
 	 */
-	if (mcip->mci_nvids > 0)
+	if (mcip->mci_nvids > 1)
 		return (B_FALSE);
 
 	/*
@@ -1608,8 +1612,8 @@
 	i_mac_perim_exit(mip);
 
 	/*
-	 * If we're changing the rx function on the primary mac of a vnic,
-	 * make sure any secondary macs on the vnic are updated as well.
+	 * If we're changing the Rx function on the primary MAC of a VNIC,
+	 * make sure any secondary addresses on the VNIC are updated as well.
 	 */
 	if (umip != NULL) {
 		ASSERT((umip->mi_state_flags & MIS_IS_VNIC) != 0);
@@ -1787,6 +1791,14 @@
 				}
 			/* Let check if we can give this an excl group */
 			} else if (group == defgrp) {
+				/*
+				 * If multiple clients share an
+				 * address then they must stay on the
+				 * default group.
+				 */
+				if (mac_check_macaddr_shared(mcip->mci_unicast))
+					return (0);
+
 				ngrp = mac_reserve_rx_group(mcip, mac_addr,
 				    B_TRUE);
 				/* Couldn't give it a group, that's fine */
@@ -1809,6 +1821,16 @@
 		}
 
 		if (group == defgrp && ((mrp->mrp_nrxrings > 0) || unspec)) {
+			/*
+			 * We are requesting Rx rings. Try to reserve
+			 * a non-default group.
+			 *
+			 * If multiple clients share an address then
+			 * they must stay on the default group.
+			 */
+			if (mac_check_macaddr_shared(mcip->mci_unicast))
+				return (EINVAL);
+
 			ngrp = mac_reserve_rx_group(mcip, mac_addr, B_TRUE);
 			if (ngrp == NULL)
 				return (ENOSPC);
@@ -2166,10 +2188,10 @@
 		flent_flags = FLOW_VNIC_MAC;
 
 	/*
-	 * For the first flow we use the mac client's name - mci_name, for
-	 * subsequent ones we just create a name with the vid. This is
+	 * For the first flow we use the MAC client's name - mci_name, for
+	 * subsequent ones we just create a name with the VID. This is
 	 * so that we can add these flows to the same flow table. This is
-	 * fine as the flow name (except for the one with the mac client's
+	 * fine as the flow name (except for the one with the MAC client's
 	 * name) is not visible. When the first flow is removed, we just replace
 	 * its fdesc with another from the list, so we will still retain the
 	 * flent with the MAC client's flow name.
@@ -2327,6 +2349,7 @@
 		 * The unicast MAC address must have been added successfully.
 		 */
 		ASSERT(mcip->mci_unicast != NULL);
+
 		/*
 		 * Push down the sub-flows that were defined on this link
 		 * hitherto. The flows are added to the active flow table
@@ -2338,15 +2361,23 @@
 
 		ASSERT(!no_unicast);
 		/*
-		 * A unicast flow already exists for that MAC client,
-		 * this flow must be the same mac address but with
-		 * different VID. It has been checked by mac_addr_in_use().
+		 * A unicast flow already exists for that MAC client
+		 * so this flow must be the same MAC address but with
+		 * a different VID. It has been checked by
+		 * mac_addr_in_use().
 		 *
-		 * We will use the SRS etc. from the mci_flent. Note that
-		 * We don't need to create kstat for this as except for
-		 * the fdesc, everything will be used from in the 1st flent.
+		 * We will use the SRS etc. from the initial
+		 * mci_flent. We don't need to create a kstat for
+		 * this, as except for the fdesc, everything will be
+		 * used from the first flent.
+		 *
+		 * The only time we should see multiple flents on the
+		 * same MAC client is on the sun4v vsw. If we removed
+		 * that code we should be able to remove the entire
+		 * notion of multiple flents on a MAC client (this
+		 * doesn't affect sub/user flows because they have
+		 * their own list unrelated to mci_flent_list).
 		 */
-
 		if (bcmp(mac_addr, map->ma_addr, map->ma_len) != 0) {
 			err = EINVAL;
 			goto bail;
@@ -2475,8 +2506,12 @@
 	boolean_t		is_vnic_primary =
 	    (flags & MAC_UNICAST_VNIC_PRIMARY);
 
-	/* when VID is non-zero, the underlying MAC can not be VNIC */
-	ASSERT(!((mip->mi_state_flags & MIS_IS_VNIC) && (vid != 0)));
+	/*
+	 * When the VID is non-zero the underlying MAC cannot be a
+	 * VNIC. I.e., dladm create-vlan cannot take a VNIC as
+	 * argument, only the primary MAC client.
+	 */
+	ASSERT(!((mip->mi_state_flags & MIS_IS_VNIC) && (vid != VLAN_ID_NONE)));
 
 	/*
 	 * Can't unicast add if the client asked only for minimal datapath
@@ -2489,18 +2524,19 @@
 	 * Check for an attempted use of the current Port VLAN ID, if enabled.
 	 * No client may use it.
 	 */
-	if (mip->mi_pvid != 0 && vid == mip->mi_pvid)
+	if (mip->mi_pvid != VLAN_ID_NONE && vid == mip->mi_pvid)
 		return (EBUSY);
 
 	/*
 	 * Check whether it's the primary client and flag it.
 	 */
-	if (!(mcip->mci_state_flags & MCIS_IS_VNIC) && is_primary && vid == 0)
+	if (!(mcip->mci_state_flags & MCIS_IS_VNIC) && is_primary &&
+	    vid == VLAN_ID_NONE)
 		mcip->mci_flags |= MAC_CLIENT_FLAGS_PRIMARY;
 
 	/*
 	 * is_vnic_primary is true when we come here as a VLAN VNIC
-	 * which uses the primary mac client's address but with a non-zero
+	 * which uses the primary MAC client's address but with a non-zero
 	 * VID. In this case the MAC address is not specified by an upper
 	 * MAC client.
 	 */
@@ -2552,7 +2588,7 @@
 		/*
 		 * Create a handle for vid 0.
 		 */
-		ASSERT(vid == 0);
+		ASSERT(vid == VLAN_ID_NONE);
 		muip = kmem_zalloc(sizeof (mac_unicast_impl_t), KM_SLEEP);
 		muip->mui_vid = vid;
 		*mah = (mac_unicast_handle_t)muip;
@@ -2572,7 +2608,9 @@
 	}
 
 	/*
-	 * If this is a VNIC/VLAN, disable softmac fast-path.
+	 * If this is a VNIC/VLAN, disable softmac fast-path. This is
+	 * only relevant to legacy devices which use softmac to
+	 * interface with GLDv3.
 	 */
 	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
 		err = mac_fastpath_disable((mac_handle_t)mip);
@@ -2620,9 +2658,11 @@
 		(void) mac_client_set_resources(mch, mrp);
 	} else if (mcip->mci_state_flags & MCIS_IS_VNIC) {
 		/*
-		 * This is a primary VLAN client, we don't support
-		 * specifying rings property for this as it inherits the
-		 * rings property from its MAC.
+		 * This is a VLAN client sharing the address of the
+		 * primary MAC client; i.e., one created via dladm
+		 * create-vlan. We don't support specifying ring
+		 * properties for this type of client as it inherits
+		 * these from the primary MAC client.
 		 */
 		if (is_vnic_primary) {
 			mac_resource_props_t	*vmrp;
@@ -2681,7 +2721,7 @@
 
 	/*
 	 * Set the flags here so that if this is a passive client, we
-	 * can return  and set it when we call mac_client_datapath_setup
+	 * can return and set it when we call mac_client_datapath_setup
 	 * when this becomes the active client. If we defer to using these
 	 * flags to mac_client_datapath_setup, then for a passive client,
 	 * we'd have to store the flags somewhere (probably fe_flags)
@@ -2984,7 +3024,7 @@
 	i_mac_perim_enter(mip);
 	if (mcip->mci_flags & MAC_CLIENT_FLAGS_VNIC_PRIMARY) {
 		/*
-		 * Called made by the upper MAC client of a VNIC.
+		 * Call made by the upper MAC client of a VNIC.
 		 * There's nothing much to do, the unicast address will
 		 * be removed by the VNIC driver when the VNIC is deleted,
 		 * but let's ensure that all our transmit is done before
@@ -2991,7 +3031,7 @@
 		 * the client does a mac_client_stop lest it trigger an
 		 * assert in the driver.
 		 */
-		ASSERT(muip->mui_vid == 0);
+		ASSERT(muip->mui_vid == VLAN_ID_NONE);
 
 		mac_tx_client_flush(mcip);
 
@@ -3055,6 +3095,7 @@
 		i_mac_perim_exit(mip);
 		return (0);
 	}
+
 	/*
 	 * Remove the VID from the list of client's VIDs.
 	 */
@@ -3081,7 +3122,7 @@
 		 * flows.
 		 */
 		flent = mac_client_get_flow(mcip, muip);
-		ASSERT(flent != NULL);
+		VERIFY3P(flent, !=, NULL);
 
 		/*
 		 * The first one is disappearing, need to make sure
@@ -3109,6 +3150,7 @@
 
 		FLOW_FINAL_REFRELE(flent);
 		ASSERT(!(mcip->mci_state_flags & MCIS_EXCLUSIVE));
+
 		/*
 		 * Enable fastpath if this is a VNIC or a VLAN.
 		 */
@@ -3122,7 +3164,8 @@
 	mui_vid = muip->mui_vid;
 	mac_client_datapath_teardown(mch, muip, flent);
 
-	if ((mcip->mci_flags & MAC_CLIENT_FLAGS_PRIMARY) && mui_vid == 0) {
+	if ((mcip->mci_flags & MAC_CLIENT_FLAGS_PRIMARY) &&
+	    mui_vid == VLAN_ID_NONE) {
 		mcip->mci_flags &= ~MAC_CLIENT_FLAGS_PRIMARY;
 	} else {
 		i_mac_perim_exit(mip);
