From 1575dddac2037ec652c4b8a76cd112eeff23ddfe Mon Sep 17 00:00:00 2001 From: Ryan Zezeski Date: Sun, 11 Jan 2026 15:25:27 -0500 Subject: [PATCH] 17813 mac softring polling does not enforce byte limit 17817 mac fanout rbytes stat always zero 17818 s_ring_rx_arg2 not set for TCP6 softrings 17823 ip_squeue_add_ring only configures IPv4 accept logic 17825 assertion tripped in tcp_update_lso() 17826 mci_resource_arg overwritten when running dualstack Change-Id: I6a6a636c3f4b6f8282e08f72fa919db9ce5c50b6 --- usr/src/uts/common/inet/ip.h | 4 +- usr/src/uts/common/inet/ip/ip_squeue.c | 12 +- usr/src/uts/common/inet/squeue.c | 13 +- usr/src/uts/common/io/aggr/aggr_grp.c | 29 +- usr/src/uts/common/io/dld/dld_proto.c | 25 +- usr/src/uts/common/io/mac/mac.c | 23 +- usr/src/uts/common/io/mac/mac_client.c | 39 +- .../uts/common/io/mac/mac_datapath_setup.c | 388 +++++++++--------- usr/src/uts/common/io/mac/mac_sched.c | 95 ++--- usr/src/uts/common/io/mac/mac_soft_ring.c | 37 +- usr/src/uts/common/io/mac/mac_stat.c | 19 +- usr/src/uts/common/sys/mac.h | 14 +- usr/src/uts/common/sys/mac_client_impl.h | 22 +- usr/src/uts/common/sys/mac_client_priv.h | 14 +- usr/src/uts/common/sys/mac_soft_ring.h | 35 +- usr/src/uts/intel/ip/ip.global-objs.debug64 | 2 + usr/src/uts/intel/ip/ip.global-objs.obj64 | 2 + 17 files changed, 434 insertions(+), 339 deletions(-) diff --git a/usr/src/uts/common/inet/ip.h b/usr/src/uts/common/inet/ip.h index e52adb9241..d28066fb96 100644 --- a/usr/src/uts/common/inet/ip.h +++ b/usr/src/uts/common/inet/ip.h @@ -25,7 +25,7 @@ * Copyright 2017 Nexenta Systems, Inc. * Copyright 2017 OmniTI Computer Consulting, Inc. All rights reserved. * Copyright 2019, Joyent, Inc. - * Copyright 2024 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ #ifndef _INET_IP_H @@ -3295,6 +3295,8 @@ extern int ip_total_hdrs_len_v4(const ip_pkt_t *); extern mblk_t *ip_accept_tcp(ill_t *, ill_rx_ring_t *, squeue_t *, mblk_t *, mblk_t **, uint_t *cnt); +extern mblk_t *ip_accept_tcp_v6(ill_t *, ill_rx_ring_t *, squeue_t *, + mblk_t *, mblk_t **, uint_t *cnt); extern void ip_rput_dlpi(ill_t *, mblk_t *); extern void ip_rput_notdata(ill_t *, mblk_t *); diff --git a/usr/src/uts/common/inet/ip/ip_squeue.c b/usr/src/uts/common/inet/ip/ip_squeue.c index 13e961333c..faef5e79ee 100644 --- a/usr/src/uts/common/inet/ip/ip_squeue.c +++ b/usr/src/uts/common/inet/ip/ip_squeue.c @@ -22,6 +22,7 @@ * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2017 Joyent, Inc. + * Copyright 2026 Oxide Computer Company */ /* @@ -247,9 +248,11 @@ ip_squeue_set_create(processorid_t id) } /* - * Called by ill_ring_add() to find an squeue to associate with a new ring. + * Obtain a free squeue and set its worker and poll thread priorities, if + * required. A free squeue is one that is not already bound to an ill_t + * and that is not marked as "default". If a free squeue does not exist, + * then one is created. */ - squeue_t * ip_squeue_getfree(pri_t pri) { @@ -499,8 +502,9 @@ ip_squeue_add_ring(ill_t *ill, void *mrp) bzero(rx_ring, sizeof (ill_rx_ring_t)); rx_ring->rr_rx = mrfp->mrf_receive; - /* XXX: Hard code it to tcp accept for now */ - rx_ring->rr_ip_accept = (ip_accept_t)ip_accept_tcp; + rx_ring->rr_ip_accept = (ill->ill_isv6 != 0) ? + (ip_accept_t)ip_accept_tcp_v6 : + (ip_accept_t)ip_accept_tcp; rx_ring->rr_intr_handle = mrfp->mrf_intr_handle; rx_ring->rr_intr_enable = (ip_mac_intr_enable_t)mrfp->mrf_intr_enable; diff --git a/usr/src/uts/common/inet/squeue.c b/usr/src/uts/common/inet/squeue.c index 5e09f00790..837e271446 100644 --- a/usr/src/uts/common/inet/squeue.c +++ b/usr/src/uts/common/inet/squeue.c @@ -24,6 +24,7 @@ /* * Copyright 2017 Joyent, Inc. + * Copyright 2026 Oxide Computer Company */ /* @@ -150,12 +151,14 @@ static uint_t squeue_drain_ns = 0; uintptr_t squeue_drain_stack_needed = 10240; uint_t squeue_drain_stack_toodeep; -#define MAX_BYTES_TO_PICKUP 150000 +/* + * The number of bytes the squeue is allowed to poll from the softring in a + * single read. The accounting is done on a per-mblk basis, so the squeue may + * poll one mblk/MTU worth of data over the limit. + */ +size_t squeue_poll_budget_bytes = 150000; #define ENQUEUE_CHAIN(sqp, mp, tail, cnt) { \ - /* \ - * Enqueue our mblk chain. \ - */ \ ASSERT(MUTEX_HELD(&(sqp)->sq_lock)); \ \ if ((sqp)->sq_last != NULL) \ @@ -956,7 +959,7 @@ poll_again: sq_mac_handle = sq_rx_ring->rr_rx_handle; ip_accept = sq_rx_ring->rr_ip_accept; sq_ill = sq_rx_ring->rr_ill; - bytes_to_pickup = MAX_BYTES_TO_PICKUP; + bytes_to_pickup = squeue_poll_budget_bytes; mutex_exit(lock); head = sq_get_pkts(sq_mac_handle, bytes_to_pickup); mp = NULL; diff --git a/usr/src/uts/common/io/aggr/aggr_grp.c b/usr/src/uts/common/io/aggr/aggr_grp.c index d3ce0877b1..197b5e2961 100644 --- a/usr/src/uts/common/io/aggr/aggr_grp.c +++ b/usr/src/uts/common/io/aggr/aggr_grp.c @@ -23,6 +23,7 @@ * Copyright 2020 Joyent, Inc. * Copyright 2020 RackTop Systems, Inc. * Copyright 2024 MNX Cloud, Inc. + * Copyright 2026 Oxide Computer Company */ /* @@ -3050,6 +3051,7 @@ aggr_grp_capab_set(aggr_grp_t *grp) grp->lg_lso = B_TRUE; grp->lg_cap_lso.lso_flags = (t_uscalar_t)-1; grp->lg_cap_lso.lso_basic_tcp_ipv4.lso_max = (t_uscalar_t)-1; + grp->lg_cap_lso.lso_basic_tcp_ipv6.lso_max = (t_uscalar_t)-1; for (port = grp->lg_ports; port != NULL; port = port->lp_next) { if (!mac_capab_get(port->lp_mh, MAC_CAPAB_HCKSUM, &cksum)) @@ -3066,10 +3068,22 @@ aggr_grp_capab_set(aggr_grp_t *grp) mac_capab_get(port->lp_mh, MAC_CAPAB_LSO, &cap_lso); if (grp->lg_lso) { grp->lg_cap_lso.lso_flags &= cap_lso.lso_flags; + + /* + * Make sure the maximum LSO for the aggr is clamped to + * the largest _common_ value across all the ports. + */ if (grp->lg_cap_lso.lso_basic_tcp_ipv4.lso_max > - cap_lso.lso_basic_tcp_ipv4.lso_max) + cap_lso.lso_basic_tcp_ipv4.lso_max) { grp->lg_cap_lso.lso_basic_tcp_ipv4.lso_max = cap_lso.lso_basic_tcp_ipv4.lso_max; + } + + if (grp->lg_cap_lso.lso_basic_tcp_ipv6.lso_max > + cap_lso.lso_basic_tcp_ipv6.lso_max) { + grp->lg_cap_lso.lso_basic_tcp_ipv6.lso_max = + cap_lso.lso_basic_tcp_ipv6.lso_max; + } } } } @@ -3108,11 +3122,20 @@ aggr_grp_capab_check(aggr_grp_t *grp, aggr_port_t *port) if (mac_capab_get(port->lp_mh, MAC_CAPAB_LSO, &cap_lso)) { if ((grp->lg_cap_lso.lso_flags & cap_lso.lso_flags) != - grp->lg_cap_lso.lso_flags) + grp->lg_cap_lso.lso_flags) { return (B_FALSE); + } + if (grp->lg_cap_lso.lso_basic_tcp_ipv4.lso_max > - cap_lso.lso_basic_tcp_ipv4.lso_max) + cap_lso.lso_basic_tcp_ipv4.lso_max) { + return (B_FALSE); + } + + if (grp->lg_cap_lso.lso_basic_tcp_ipv6.lso_max > + cap_lso.lso_basic_tcp_ipv6.lso_max) { return (B_FALSE); + } + } else { return (B_FALSE); } diff --git a/usr/src/uts/common/io/dld/dld_proto.c b/usr/src/uts/common/io/dld/dld_proto.c index 2f49c6c452..9937073c79 100644 --- a/usr/src/uts/common/io/dld/dld_proto.c +++ b/usr/src/uts/common/io/dld/dld_proto.c @@ -22,7 +22,7 @@ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2012, Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2018, Joyent, Inc. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* @@ -1432,6 +1432,8 @@ dld_capab_direct(dld_str_t *dsp, void *data, uint_t flags) static int dld_capab_poll_enable(dld_str_t *dsp, dld_capab_poll_t *poll) { + mac_resource_cb_t rcbs = { 0 }; + if (dsp->ds_polling) return (EINVAL); @@ -1454,29 +1456,28 @@ dld_capab_poll_enable(dld_str_t *dsp, dld_capab_poll_t *poll) * the user decides to modify CPU bindings to use more CPUs for the * device in which case we will switch to fanout using soft rings. */ - mac_resource_set_common(dsp->ds_mch, - (mac_resource_add_t)poll->poll_ring_add_cf, - (mac_resource_remove_t)poll->poll_ring_remove_cf, - (mac_resource_quiesce_t)poll->poll_ring_quiesce_cf, - (mac_resource_restart_t)poll->poll_ring_restart_cf, - (mac_resource_bind_t)poll->poll_ring_bind_cf, - poll->poll_ring_ch); + rcbs.mrc_add = (mac_resource_add_t)poll->poll_ring_add_cf; + rcbs.mrc_remove = (mac_resource_remove_t)poll->poll_ring_remove_cf; + rcbs.mrc_quiesce = (mac_resource_quiesce_t)poll->poll_ring_quiesce_cf; + rcbs.mrc_restart = (mac_resource_restart_t)poll->poll_ring_restart_cf; + rcbs.mrc_bind = (mac_resource_bind_t)poll->poll_ring_bind_cf; + rcbs.mrc_arg = poll->poll_ring_ch; - mac_client_poll_enable(dsp->ds_mch); + mac_resource_set(dsp->ds_mch, &rcbs, dsp->ds_sap == ETHERTYPE_IPV6); + mac_client_poll_enable(dsp->ds_mch, dsp->ds_sap == ETHERTYPE_IPV6); dsp->ds_polling = B_TRUE; return (0); } -/* ARGSUSED */ static int dld_capab_poll_disable(dld_str_t *dsp, dld_capab_poll_t *poll) { if (!dsp->ds_polling) return (EINVAL); - mac_client_poll_disable(dsp->ds_mch); - mac_resource_set(dsp->ds_mch, NULL, NULL); + mac_client_poll_disable(dsp->ds_mch, dsp->ds_sap == ETHERTYPE_IPV6); + mac_resource_clear(dsp->ds_mch, dsp->ds_sap == ETHERTYPE_IPV6); dsp->ds_polling = B_FALSE; return (0); diff --git a/usr/src/uts/common/io/mac/mac.c b/usr/src/uts/common/io/mac/mac.c index 05f35fce8d..8b8327d5f4 100644 --- a/usr/src/uts/common/io/mac/mac.c +++ b/usr/src/uts/common/io/mac/mac.c @@ -24,7 +24,7 @@ * Copyright 2020 Joyent, Inc. * Copyright 2015 Garrett D'Amore * Copyright 2020 RackTop Systems, Inc. - * Copyright 2024 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* @@ -2260,20 +2260,27 @@ mac_rx_srs_quiesce(mac_soft_ring_set_t *srs, uint_t srs_quiesce_flag) flow_entry_t *flent = srs->srs_flent; uint_t mr_flag, srs_done_flag; - ASSERT(MAC_PERIM_HELD((mac_handle_t)FLENT_TO_MIP(flent))); - ASSERT(!(srs->srs_type & SRST_TX)); + VERIFY(mac_perim_held((mac_handle_t)FLENT_TO_MIP(flent))); + VERIFY0(srs->srs_type & SRST_TX); if (srs_quiesce_flag == SRS_CONDEMNED) { mr_flag = MR_CONDEMNED; srs_done_flag = SRS_CONDEMNED_DONE; - if (srs->srs_type & SRST_CLIENT_POLL_ENABLED) - mac_srs_client_poll_disable(srs->srs_mcip, srs); + + if (srs->srs_type & SRST_CLIENT_POLL_V4) { + mac_srs_client_poll_disable(srs->srs_mcip, srs, + B_FALSE); + } + + if (srs->srs_type & SRST_CLIENT_POLL_V6) { + mac_srs_client_poll_disable(srs->srs_mcip, srs, + B_TRUE); + } } else { - ASSERT(srs_quiesce_flag == SRS_QUIESCE); + VERIFY3U(srs_quiesce_flag, ==, SRS_QUIESCE); mr_flag = MR_QUIESCE; srs_done_flag = SRS_QUIESCE_DONE; - if (srs->srs_type & SRST_CLIENT_POLL_ENABLED) - mac_srs_client_poll_quiesce(srs->srs_mcip, srs); + mac_srs_client_poll_quiesce(srs->srs_mcip, srs); } if (srs->srs_ring != NULL) { diff --git a/usr/src/uts/common/io/mac/mac_client.c b/usr/src/uts/common/io/mac/mac_client.c index 8a8b47e218..bcceca3147 100644 --- a/usr/src/uts/common/io/mac/mac_client.c +++ b/usr/src/uts/common/io/mac/mac_client.c @@ -24,7 +24,7 @@ * Copyright 2019 Joyent, Inc. * Copyright 2017 RackTop Systems. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* @@ -3945,28 +3945,29 @@ mac_notify_remove(mac_notify_handle_t mnh, boolean_t wait) * Associate resource management callbacks with the specified MAC * clients. */ - void -mac_resource_set_common(mac_client_handle_t mch, mac_resource_add_t add, - mac_resource_remove_t remove, mac_resource_quiesce_t quiesce, - mac_resource_restart_t restart, mac_resource_bind_t bind, - void *arg) +mac_resource_set(mac_client_handle_t mch, mac_resource_cb_t *rcbs, + boolean_t is_v6) { mac_client_impl_t *mcip = (mac_client_impl_t *)mch; - mcip->mci_resource_add = add; - mcip->mci_resource_remove = remove; - mcip->mci_resource_quiesce = quiesce; - mcip->mci_resource_restart = restart; - mcip->mci_resource_bind = bind; - mcip->mci_resource_arg = arg; + if (is_v6) { + mcip->mci_rcb6 = *rcbs; + } else { + mcip->mci_rcb4 = *rcbs; + } } void -mac_resource_set(mac_client_handle_t mch, mac_resource_add_t add, void *arg) +mac_resource_clear(mac_client_handle_t mch, boolean_t is_v6) { - /* update the 'resource_add' callback */ - mac_resource_set_common(mch, add, NULL, NULL, NULL, NULL, arg); + mac_client_impl_t *mcip = (mac_client_impl_t *)mch; + + if (is_v6) { + bzero(&mcip->mci_rcb6, sizeof (mcip->mci_rcb6)); + } else { + bzero(&mcip->mci_rcb4, sizeof (mcip->mci_rcb4)); + } } /* @@ -3974,7 +3975,7 @@ mac_resource_set(mac_client_handle_t mch, mac_resource_add_t add, void *arg) * SRS's and the soft rings of the client */ void -mac_client_poll_enable(mac_client_handle_t mch) +mac_client_poll_enable(mac_client_handle_t mch, boolean_t is_v6) { mac_client_impl_t *mcip = (mac_client_impl_t *)mch; mac_soft_ring_set_t *mac_srs; @@ -3988,7 +3989,7 @@ mac_client_poll_enable(mac_client_handle_t mch) for (i = 0; i < flent->fe_rx_srs_cnt; i++) { mac_srs = (mac_soft_ring_set_t *)flent->fe_rx_srs[i]; ASSERT(mac_srs->srs_mcip == mcip); - mac_srs_client_poll_enable(mcip, mac_srs); + mac_srs_client_poll_enable(mcip, mac_srs, is_v6); } } @@ -3997,7 +3998,7 @@ mac_client_poll_enable(mac_client_handle_t mch) * the SRS's and the soft rings of the client */ void -mac_client_poll_disable(mac_client_handle_t mch) +mac_client_poll_disable(mac_client_handle_t mch, boolean_t is_v6) { mac_client_impl_t *mcip = (mac_client_impl_t *)mch; mac_soft_ring_set_t *mac_srs; @@ -4011,7 +4012,7 @@ mac_client_poll_disable(mac_client_handle_t mch) for (i = 0; i < flent->fe_rx_srs_cnt; i++) { mac_srs = (mac_soft_ring_set_t *)flent->fe_rx_srs[i]; ASSERT(mac_srs->srs_mcip == mcip); - mac_srs_client_poll_disable(mcip, mac_srs); + mac_srs_client_poll_disable(mcip, mac_srs, is_v6); } } diff --git a/usr/src/uts/common/io/mac/mac_datapath_setup.c b/usr/src/uts/common/io/mac/mac_datapath_setup.c index 8abeb178ed..22b70b65fe 100644 --- a/usr/src/uts/common/io/mac/mac_datapath_setup.c +++ b/usr/src/uts/common/io/mac/mac_datapath_setup.c @@ -22,7 +22,7 @@ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2018 Joyent, Inc. * Copyright 2020 RackTop Systems. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ #include @@ -308,196 +308,240 @@ mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) /* POLLING SETUP AND TEAR DOWN ROUTINES */ /* - * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart - * - * These routines are used to call back into the upper layer - * (primarily TCP squeue) to stop polling the soft rings or - * restart polling. + * Quiesce polling on the TCP/IP squeues. */ void -mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, - mac_soft_ring_set_t *mac_srs) +mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, mac_soft_ring_set_t *srs) { - mac_soft_ring_t *softring; + VERIFY(mac_perim_held((mac_handle_t)mcip->mci_mip)); - ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); + if (srs->srs_type & SRST_CLIENT_POLL_V4) { + for (uint_t i = 0; i < srs->srs_tcp_ring_count; i++) { + mac_soft_ring_t *sr = srs->srs_tcp_soft_rings[i]; - if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { - ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); - return; + if (sr->s_ring_rx_arg2 != NULL) { + mcip->mci_rcb4.mrc_quiesce( + mcip->mci_rcb4.mrc_arg, sr->s_ring_rx_arg2); + } + } } - for (softring = mac_srs->srs_soft_ring_head; - softring != NULL; softring = softring->s_ring_next) { - if ((softring->s_ring_type & ST_RING_TCP) && - (softring->s_ring_rx_arg2 != NULL)) { - mcip->mci_resource_quiesce(mcip->mci_resource_arg, - softring->s_ring_rx_arg2); + if (srs->srs_type & SRST_CLIENT_POLL_V6) { + for (uint_t i = 0; i < srs->srs_tcp6_ring_count; i++) { + mac_soft_ring_t *sr = srs->srs_tcp6_soft_rings[i]; + + if (sr->s_ring_rx_arg2 != NULL) { + mcip->mci_rcb6.mrc_quiesce( + mcip->mci_rcb6.mrc_arg, sr->s_ring_rx_arg2); + } } } } +/* + * Restart polling on the TCP/IP squeues. + */ void -mac_srs_client_poll_restart(mac_client_impl_t *mcip, - mac_soft_ring_set_t *mac_srs) +mac_srs_client_poll_restart(mac_client_impl_t *mcip, mac_soft_ring_set_t *srs) { - mac_soft_ring_t *softring; + VERIFY(mac_perim_held((mac_handle_t)mcip->mci_mip)); - ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); + if (srs->srs_type & SRST_CLIENT_POLL_V4) { + for (uint_t i = 0; i < srs->srs_tcp_ring_count; i++) { + mac_soft_ring_t *sr = srs->srs_tcp_soft_rings[i]; - if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { - ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); - return; + if (sr->s_ring_rx_arg2 != NULL) { + mcip->mci_rcb4.mrc_restart( + mcip->mci_rcb4.mrc_arg, sr->s_ring_rx_arg2); + } + } } - for (softring = mac_srs->srs_soft_ring_head; - softring != NULL; softring = softring->s_ring_next) { - if ((softring->s_ring_type & ST_RING_TCP) && - (softring->s_ring_rx_arg2 != NULL)) { - mcip->mci_resource_restart(mcip->mci_resource_arg, - softring->s_ring_rx_arg2); + if (srs->srs_type & SRST_CLIENT_POLL_V6) { + for (uint_t i = 0; i < srs->srs_tcp6_ring_count; i++) { + mac_soft_ring_t *sr = srs->srs_tcp6_soft_rings[i]; + + if (sr->s_ring_rx_arg2 != NULL) { + mcip->mci_rcb6.mrc_restart( + mcip->mci_rcb6.mrc_arg, sr->s_ring_rx_arg2); + } } } } +static void +mac_srs_client_poll_enable_i(mac_soft_ring_set_t *srs, uint_t sr_cnt, + mac_soft_ring_t **udp_rings, mac_soft_ring_t **tcp_rings, + mac_direct_rx_t drx, void *drx_arg, mac_resource_cb_t *rcb) +{ + /* + * TCP and UDP support DLS bypass. Squeue polling support implies DLS + * bypass since the squeue poll path does not have DLS processing. + */ + for (uint_t i = 0; i < sr_cnt; i++) { + mac_soft_ring_dls_bypass_enable(udp_rings[i], drx, drx_arg); + } + + for (uint_t i = 0; i < sr_cnt; i++) { + mac_soft_ring_t *tcp_sr = tcp_rings[i]; + mac_rx_fifo_t mrf; + + /* + * Polling should be configured only once on a given + * softring. + */ + VERIFY3P(tcp_sr->s_ring_rx_arg2, ==, NULL); + + mac_soft_ring_dls_bypass_enable(tcp_sr, drx, drx_arg); + + bzero(&mrf, sizeof (mrf)); + mrf.mrf_type = MAC_RX_FIFO; + mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; + mrf.mrf_intr_enable = + (mac_intr_enable_t)mac_soft_ring_intr_enable; + mrf.mrf_intr_disable = + (mac_intr_disable_t)mac_soft_ring_intr_disable; + mrf.mrf_rx_arg = tcp_sr; + mrf.mrf_intr_handle = (mac_intr_handle_t)tcp_sr; + mrf.mrf_cpu_id = tcp_sr->s_ring_cpuid; + mrf.mrf_flow_priority = srs->srs_pri; + + tcp_sr->s_ring_rx_arg2 = rcb->mrc_add(rcb->mrc_arg, + (mac_resource_t *)&mrf); + } +} + /* * Register the given SRS and associated soft rings with the consumer and * enable the polling interface used by the consumer.(i.e IP) over this * SRS and associated soft rings. */ void -mac_srs_client_poll_enable(mac_client_impl_t *mcip, - mac_soft_ring_set_t *mac_srs) +mac_srs_client_poll_enable(mac_client_impl_t *mcip, mac_soft_ring_set_t *srs, + boolean_t is_v6) { - mac_rx_fifo_t mrf; - mac_soft_ring_t *softring; - - ASSERT(mac_srs->srs_mcip == mcip); - ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); + VERIFY3P(srs->srs_mcip, ==, mcip); + VERIFY(mac_perim_held((mac_handle_t)mcip->mci_mip)); if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) return; - bzero(&mrf, sizeof (mac_rx_fifo_t)); - mrf.mrf_type = MAC_RX_FIFO; - /* * A SRS is capable of acting as a soft ring for cases * where no fanout is needed. This is the case for userland * flows. */ - if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) + if (srs->srs_type & SRST_NO_SOFT_RINGS) return; - mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; - mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; - mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; - mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; + if (is_v6) { + mac_srs_client_poll_enable_i(srs, srs->srs_tcp_ring_count, + srs->srs_udp6_soft_rings, srs->srs_tcp6_soft_rings, + mcip->mci_direct_rx.mdrx_v6, + mcip->mci_direct_rx.mdrx_arg_v6, &mcip->mci_rcb6); - softring = mac_srs->srs_soft_ring_head; - while (softring != NULL) { - if (mcip->mci_direct_rx.mdrx_v4 != NULL && - softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { - /* - * TCP and UDP support DLS bypass. Squeue polling - * support implies DLS bypass since the squeue poll - * path does not have DLS processing. - */ - mac_soft_ring_dls_bypass(softring, - mcip->mci_direct_rx.mdrx_v4, - mcip->mci_direct_rx.mdrx_arg_v4); - } - if (mcip->mci_direct_rx.mdrx_v6 != NULL && - softring->s_ring_type & (ST_RING_TCP6 | ST_RING_UDP6)) { - /* - * TCP and UDP support DLS bypass. Squeue polling - * support implies DLS bypass since the squeue poll - * path does not have DLS processing. - */ - mac_soft_ring_dls_bypass(softring, - mcip->mci_direct_rx.mdrx_v6, - mcip->mci_direct_rx.mdrx_arg_v6); - } + mutex_enter(&srs->srs_lock); + srs->srs_type |= (SRST_CLIENT_POLL_V6 | SRST_DLS_BYPASS_V6); + mutex_exit(&srs->srs_lock); + } else { + mac_srs_client_poll_enable_i(srs, srs->srs_tcp_ring_count, + srs->srs_udp_soft_rings, srs->srs_tcp_soft_rings, + mcip->mci_direct_rx.mdrx_v4, + mcip->mci_direct_rx.mdrx_arg_v4, &mcip->mci_rcb4); + + mutex_enter(&srs->srs_lock); + srs->srs_type |= (SRST_CLIENT_POLL_V4 | SRST_DLS_BYPASS_V4); + mutex_exit(&srs->srs_lock); + } +} + +static void +mac_srs_client_poll_disable_i(mac_client_impl_t *mcip, uint_t sr_cnt, + mac_soft_ring_t **udp_rings, mac_soft_ring_t **tcp_rings, + mac_resource_cb_t *rcb) +{ + for (uint_t i = 0; i < sr_cnt; i++) { + mac_soft_ring_t *tcp_sr = tcp_rings[i]; /* - * Non-TCP protocols don't support squeues. Hence we don't - * make any ring addition callbacks for non-TCP rings + * Remove the IP ring if there is one associated with this + * softring. Note that IP rings are a limited resource; and + * SRST_CLIENT_POLL_V4/V6 being set on the SRS is no + * guarantee that all TCP softrings have an associated IP + * ring. This is by design. See ip_squeue_add_ring(). */ - if (!(softring->s_ring_type & ST_RING_TCP)) { - softring->s_ring_rx_arg2 = NULL; - softring = softring->s_ring_next; - continue; + if (tcp_sr->s_ring_rx_arg2 != NULL) { + VERIFY3P(rcb->mrc_arg, !=, NULL); + rcb->mrc_remove(rcb->mrc_arg, tcp_sr->s_ring_rx_arg2); + tcp_sr->s_ring_rx_arg2 = NULL; } - mrf.mrf_rx_arg = softring; - mrf.mrf_intr_handle = (mac_intr_handle_t)softring; - mrf.mrf_cpu_id = softring->s_ring_cpuid; - mrf.mrf_flow_priority = mac_srs->srs_pri; + mac_soft_ring_dls_bypass_disable(tcp_sr, mcip); + } - softring->s_ring_rx_arg2 = mcip->mci_resource_add( - mcip->mci_resource_arg, (mac_resource_t *)&mrf); + for (uint_t i = 0; i < sr_cnt; i++) { + mac_soft_ring_t *udp_sr = udp_rings[i]; - softring = softring->s_ring_next; + /* There is no polling on UDP; this should always be NULL. */ + VERIFY3P(udp_sr->s_ring_rx_arg2, ==, NULL); + mac_soft_ring_dls_bypass_disable(udp_sr, mcip); } } /* * Unregister the given SRS and associated soft rings with the consumer and - * disable the polling interface used by the consumer.(i.e IP) over this + * disable the polling interface used by the consumer (i.e IP) over this * SRS and associated soft rings. */ void -mac_srs_client_poll_disable(mac_client_impl_t *mcip, - mac_soft_ring_set_t *mac_srs) +mac_srs_client_poll_disable(mac_client_impl_t *mcip, mac_soft_ring_set_t *srs, + boolean_t is_v6) { - mac_soft_ring_t *softring; - - ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); + VERIFY(mac_perim_held((mac_handle_t)mcip->mci_mip)); /* * A SRS is capable of acting as a soft ring for cases * where no protocol fanout is needed. This is the case * for userland flows. Nothing to do here. */ - if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) + if (srs->srs_type & SRST_NO_SOFT_RINGS) return; - mutex_enter(&mac_srs->srs_lock); - if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { - ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); - mutex_exit(&mac_srs->srs_lock); + mutex_enter(&srs->srs_lock); + if (!is_v6 && !(srs->srs_type & SRST_CLIENT_POLL_V4)) { + VERIFY(!(srs->srs_type & SRST_DLS_BYPASS_V4)); + mutex_exit(&srs->srs_lock); + return; + } + + if (is_v6 && !(srs->srs_type & SRST_CLIENT_POLL_V6)) { + VERIFY(!(srs->srs_type & SRST_DLS_BYPASS_V6)); + mutex_exit(&srs->srs_lock); return; } - mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); - mutex_exit(&mac_srs->srs_lock); /* - * DLS bypass is now disabled in the case of both TCP and UDP. - * Reset the soft ring callbacks to the standard 'mac_rx_deliver' - * callback. In addition, in the case of TCP, invoke IP's callback - * for ring removal. + * Before modifying TCP/UDP softring state we must first inform the SRS + * that DLS bypass is no longer to be performed; thereby directing all + * future traffic to the OTH softring. */ - for (softring = mac_srs->srs_soft_ring_head; - softring != NULL; softring = softring->s_ring_next) { - if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) - continue; + if (is_v6) { + srs->srs_type &= ~(SRST_CLIENT_POLL_V6 | + SRST_DLS_BYPASS_V6); + } else { + srs->srs_type &= ~(SRST_CLIENT_POLL_V4 | + SRST_DLS_BYPASS_V4); + } - if ((softring->s_ring_type & ST_RING_TCP) && - softring->s_ring_rx_arg2 != NULL) { - mcip->mci_resource_remove(mcip->mci_resource_arg, - softring->s_ring_rx_arg2); - } + mutex_exit(&srs->srs_lock); - mutex_enter(&softring->s_ring_lock); - while (softring->s_ring_state & S_RING_PROC) { - softring->s_ring_state |= S_RING_CLIENT_WAIT; - cv_wait(&softring->s_ring_client_cv, - &softring->s_ring_lock); - } - softring->s_ring_state &= ~S_RING_CLIENT_WAIT; - softring->s_ring_rx_arg2 = NULL; - softring->s_ring_rx_func = mac_rx_deliver; - softring->s_ring_rx_arg1 = mcip; - mutex_exit(&softring->s_ring_lock); + if (is_v6) { + mac_srs_client_poll_disable_i(mcip, srs->srs_tcp_ring_count, + srs->srs_udp6_soft_rings, srs->srs_tcp6_soft_rings, + &mcip->mci_rcb6); + } else { + mac_srs_client_poll_disable_i(mcip, srs->srs_tcp_ring_count, + srs->srs_udp_soft_rings, srs->srs_tcp_soft_rings, + &mcip->mci_rcb4); } } @@ -1511,11 +1555,6 @@ mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { /* Reset bandwidth limit */ if (srs->srs_type & SRST_BW_CONTROL) { - softring = srs->srs_soft_ring_head; - while (softring != NULL) { - softring->s_ring_type &= ~ST_RING_BW_CTL; - softring = softring->s_ring_next; - } srs->srs_type &= ~SRST_BW_CONTROL; srs->srs_drain_func = mac_rx_srs_drain; } @@ -1529,11 +1568,6 @@ mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) srs->srs_bw->mac_bw_drop_threshold = srs->srs_bw->mac_bw_limit << 1; if (!(srs->srs_type & SRST_BW_CONTROL)) { - softring = srs->srs_soft_ring_head; - while (softring != NULL) { - softring->s_ring_type |= ST_RING_BW_CTL; - softring = softring->s_ring_next; - } srs->srs_type |= SRST_BW_CONTROL; srs->srs_drain_func = mac_rx_srs_drain_bw; } @@ -1735,10 +1769,9 @@ mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) } void -mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, - mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, - processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, - mac_resource_handle_t x_arg2, boolean_t set_bypass) +mac_srs_create_proto_softrings(int id, pri_t pri, mac_client_impl_t *mcip, + mac_soft_ring_set_t *mac_srs, processorid_t cpuid, mac_direct_rx_t rx_func, + void *x_arg1, mac_resource_handle_t x_arg2, boolean_t set_bypass) { mac_soft_ring_t *softring; mac_rx_fifo_t mrf; @@ -1751,7 +1784,7 @@ mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, mrf.mrf_flow_priority = pri; softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, - (type|ST_RING_TCP), pri, mcip, mac_srs, + ST_RING_TCP, pri, mcip, mac_srs, cpuid, rx_func, x_arg1, x_arg2); softring->s_ring_rx_arg2 = NULL; @@ -1760,8 +1793,8 @@ mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, * squeue can also poll their corresponding soft rings. */ if (set_bypass && mcip->mci_direct_rx.mdrx_v4 != NULL && - (mcip->mci_resource_arg != NULL)) { - mac_soft_ring_dls_bypass(softring, + (mcip->mci_rcb4.mrc_arg != NULL)) { + mac_soft_ring_dls_bypass_enable(softring, mcip->mci_direct_rx.mdrx_v4, mcip->mci_direct_rx.mdrx_arg_v4); @@ -1775,9 +1808,8 @@ mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, * the softring so the flow control can be pushed * all the way to H/W. */ - softring->s_ring_rx_arg2 = - mcip->mci_resource_add((void *)mcip->mci_resource_arg, - (mac_resource_t *)&mrf); + softring->s_ring_rx_arg2 = mcip->mci_rcb4.mrc_add( + mcip->mci_rcb4.mrc_arg, (mac_resource_t *)&mrf); } /* @@ -1787,53 +1819,49 @@ mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, * bypass the DLS layer. */ softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, - (type|ST_RING_UDP), pri, mcip, mac_srs, + ST_RING_UDP, pri, mcip, mac_srs, cpuid, rx_func, x_arg1, x_arg2); softring->s_ring_rx_arg2 = NULL; - if (set_bypass && mcip->mci_direct_rx.mdrx_v4 != NULL && - (mcip->mci_resource_arg != NULL)) { - mac_soft_ring_dls_bypass(softring, + if (set_bypass && mcip->mci_direct_rx.mdrx_v4 != NULL) { + mac_soft_ring_dls_bypass_enable(softring, mcip->mci_direct_rx.mdrx_v4, mcip->mci_direct_rx.mdrx_arg_v4); } /* TCP for IPv6. */ softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, - (type|ST_RING_TCP6), pri, mcip, mac_srs, + ST_RING_TCP6, pri, mcip, mac_srs, cpuid, rx_func, x_arg1, x_arg2); softring->s_ring_rx_arg2 = NULL; if (set_bypass && mcip->mci_direct_rx.mdrx_v6 != NULL && - (mcip->mci_resource_arg != NULL)) { - mac_soft_ring_dls_bypass(softring, + (mcip->mci_rcb6.mrc_arg != NULL)) { + mac_soft_ring_dls_bypass_enable(softring, mcip->mci_direct_rx.mdrx_v6, mcip->mci_direct_rx.mdrx_arg_v6); mrf.mrf_rx_arg = softring; mrf.mrf_intr_handle = (mac_intr_handle_t)softring; - - softring->s_ring_rx_arg2 = - mcip->mci_resource_add((void *)mcip->mci_resource_arg, - (mac_resource_t *)&mrf); + softring->s_ring_rx_arg2 = mcip->mci_rcb6.mrc_add( + mcip->mci_rcb6.mrc_arg, (mac_resource_t *)&mrf); } /* UDP for IPv6. */ softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, - (type|ST_RING_UDP6), pri, mcip, mac_srs, + ST_RING_UDP6, pri, mcip, mac_srs, cpuid, rx_func, x_arg1, x_arg2); softring->s_ring_rx_arg2 = NULL; - if (set_bypass && mcip->mci_direct_rx.mdrx_v6 != NULL && - (mcip->mci_resource_arg != NULL)) { - mac_soft_ring_dls_bypass(softring, + if (set_bypass && mcip->mci_direct_rx.mdrx_v6 != NULL) { + mac_soft_ring_dls_bypass_enable(softring, mcip->mci_direct_rx.mdrx_v6, mcip->mci_direct_rx.mdrx_arg_v6); } /* Create the Oth softrings which has to go through the DLS. */ softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, - (type|ST_RING_OTH), pri, mcip, mac_srs, + ST_RING_OTH, pri, mcip, mac_srs, cpuid, rx_func, x_arg1, x_arg2); softring->s_ring_rx_arg2 = NULL; } @@ -1852,7 +1880,6 @@ mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) { mac_soft_ring_t *softring; - uint32_t soft_ring_flag = 0; processorid_t cpuid = -1; int i, srings_present, new_fanout_cnt; mac_cpus_t *srs_cpu; @@ -1867,11 +1894,6 @@ mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, srs_cpu = &mac_rx_srs->srs_cpu; new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; - mutex_enter(&mac_rx_srs->srs_lock); - if (mac_rx_srs->srs_type & SRST_BW_CONTROL) - soft_ring_flag |= ST_RING_BW_CTL; - mutex_exit(&mac_rx_srs->srs_lock); - if (new_fanout_cnt > srings_present) { /* soft rings increased */ mutex_enter(&mac_rx_srs->srs_lock); @@ -1884,9 +1906,9 @@ mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, * Create the protocol softrings and set the * DLS bypass where possible. */ - mac_srs_create_proto_softrings(i, soft_ring_flag, - mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, - rx_func, x_arg1, x_arg2, B_TRUE); + mac_srs_create_proto_softrings(i, mac_rx_srs->srs_pri, + mcip, mac_rx_srs, cpuid, rx_func, x_arg1, x_arg2, + B_TRUE); } mac_srs_update_fanout_list(mac_rx_srs); } else if (new_fanout_cnt < srings_present) { @@ -1902,14 +1924,14 @@ mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, i < mac_rx_srs->srs_tcp_ring_count; i++) { softring = mac_rx_srs->srs_tcp_soft_rings[i]; if (softring->s_ring_rx_arg2 != NULL) { - mcip->mci_resource_remove( - (void *)mcip->mci_resource_arg, + mcip->mci_rcb4.mrc_remove( + mcip->mci_rcb4.mrc_arg, softring->s_ring_rx_arg2); } softring = mac_rx_srs->srs_tcp6_soft_rings[i]; if (softring->s_ring_rx_arg2 != NULL) { - mcip->mci_resource_remove( - (void *)mcip->mci_resource_arg, + mcip->mci_rcb6.mrc_remove( + mcip->mci_rcb6.mrc_arg, softring->s_ring_rx_arg2); } mac_soft_ring_remove(mac_rx_srs, @@ -1942,12 +1964,12 @@ mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, cpuid); softring = mac_rx_srs->srs_tcp_soft_rings[i]; if (softring->s_ring_rx_arg2 != NULL) { - mcip->mci_resource_bind((void *)mcip->mci_resource_arg, + mcip->mci_rcb4.mrc_bind(mcip->mci_rcb4.mrc_arg, softring->s_ring_rx_arg2, cpuid); } softring = mac_rx_srs->srs_tcp6_soft_rings[i]; if (softring->s_ring_rx_arg2 != NULL) { - mcip->mci_resource_bind((void *)mcip->mci_resource_arg, + mcip->mci_rcb6.mrc_bind(mcip->mci_rcb6.mrc_arg, softring->s_ring_rx_arg2, cpuid); } } @@ -1977,7 +1999,6 @@ mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, { int i; processorid_t cpuid; - uint32_t soft_ring_flag = 0; int soft_ring_cnt; mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; @@ -1990,10 +2011,6 @@ mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, mutex_exit(&mac_rx_srs->srs_lock); ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); - - if (mac_rx_srs->srs_type & SRST_BW_CONTROL) - soft_ring_flag |= ST_RING_BW_CTL; - ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; /* @@ -2010,9 +2027,9 @@ mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, for (i = 0; i < soft_ring_cnt; i++) { cpuid = srs_cpu->mc_rx_fanout_cpus[i]; /* Create the protocol softrings */ - mac_srs_create_proto_softrings(i, soft_ring_flag, - mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, - rx_func, x_arg1, x_arg2, B_FALSE); + mac_srs_create_proto_softrings(i, mac_rx_srs->srs_pri, + mcip, mac_rx_srs, cpuid, rx_func, x_arg1, x_arg2, + B_FALSE); } mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); @@ -2046,7 +2063,8 @@ alldone: if (soft_ring_cnt > 1) mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; mac_srs_update_fanout_list(mac_rx_srs); - mac_srs_client_poll_enable(mcip, mac_rx_srs); + mac_srs_client_poll_enable(mcip, mac_rx_srs, B_FALSE); + mac_srs_client_poll_enable(mcip, mac_rx_srs, B_TRUE); return; no_softrings: @@ -2054,9 +2072,8 @@ no_softrings: mutex_enter(&cpu_lock); cpuid = mac_next_bind_cpu(cpupart); /* Create the protocol softrings */ - mac_srs_create_proto_softrings(0, soft_ring_flag, - mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, - rx_func, x_arg1, x_arg2, B_FALSE); + mac_srs_create_proto_softrings(0, mac_rx_srs->srs_pri, mcip, + mac_rx_srs, cpuid, rx_func, x_arg1, x_arg2, B_FALSE); mutex_exit(&cpu_lock); } else { /* @@ -2066,7 +2083,8 @@ no_softrings: mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; } mac_srs_update_fanout_list(mac_rx_srs); - mac_srs_client_poll_enable(mcip, mac_rx_srs); + mac_srs_client_poll_enable(mcip, mac_rx_srs, B_FALSE); + mac_srs_client_poll_enable(mcip, mac_rx_srs, B_TRUE); } /* diff --git a/usr/src/uts/common/io/mac/mac_sched.c b/usr/src/uts/common/io/mac/mac_sched.c index a64f56d62d..fb8c1cf2a9 100644 --- a/usr/src/uts/common/io/mac/mac_sched.c +++ b/usr/src/uts/common/io/mac/mac_sched.c @@ -23,7 +23,7 @@ * Use is subject to license terms. * Copyright 2018 Joyent, Inc. * Copyright 2013 Nexenta Systems, Inc. All rights reserved. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* @@ -1447,7 +1447,7 @@ mac_srs_fire(void *arg) #define COMPUTE_INDEX(key, sz) (key % sz) -#define FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) { \ +#define FANOUT_ENQUEUE_MP(head, tail, cnt, sz, sz0, mp) { \ if ((tail) != NULL) { \ ASSERT((tail)->b_next == NULL); \ (tail)->b_next = (mp); \ @@ -1457,15 +1457,13 @@ mac_srs_fire(void *arg) } \ (tail) = (mp); \ (cnt)++; \ - if ((bw_ctl)) \ - (sz) += (sz0); \ + (sz) += (sz0); \ } #define MAC_FANOUT_DEFAULT 0 #define MAC_FANOUT_RND_ROBIN 1 int mac_fanout_type = MAC_FANOUT_DEFAULT; -#define MAX_SR_TYPES 5 /* fanout types for port based hashing */ typedef enum pkt_type { V4_TCP = 0, @@ -1495,15 +1493,14 @@ typedef enum pkt_type { static void mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) { - mblk_t *headmp[MAX_SR_TYPES] = { 0 }; - mblk_t *tailmp[MAX_SR_TYPES] = { 0 }; - int cnt[MAX_SR_TYPES] = { 0 }; - size_t sz[MAX_SR_TYPES] = { 0 }; + mblk_t *headmp[ST_RING_NUM_PROTO] = { 0 }; + mblk_t *tailmp[ST_RING_NUM_PROTO] = { 0 }; + int cnt[ST_RING_NUM_PROTO] = { 0 }; + size_t sz[ST_RING_NUM_PROTO] = { 0 }; mac_client_impl_t *mcip = mac_srs->srs_mcip; const boolean_t is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER); - const boolean_t bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0); /* * If we don't have a Rx ring, S/W classification would have done @@ -1516,13 +1513,16 @@ mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER; /* - * Some clients, such as non-ethernet, need DLS processing in - * the Rx path. Such clients clear the SRST_DLS_BYPASS flag. - * DLS bypass may also be disabled via the - * MCIS_RX_BYPASS_DISABLE flag. + * Some clients, such as non-ethernet, need DLS processing in the Rx + * path. Such clients clear the bypass flag. DLS bypass may also be + * disabled via the MCIS_RX_BYPASS_DISABLE flag. */ - const boolean_t dls_bypass = - ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) && + const boolean_t dls_bypass_v4 = + ((mac_srs->srs_type & SRST_DLS_BYPASS_V4) != 0) && + ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0); + + const boolean_t dls_bypass_v6 = + ((mac_srs->srs_type & SRST_DLS_BYPASS_V6) != 0) && ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0); /* @@ -1591,12 +1591,13 @@ mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) dstaddr = non_ether_mhi.mhi_daddr; } - if (!dls_bypass) { + if ((!dls_bypass_v4 && meoi.meoi_l3proto == ETHERTYPE_IP) || + (!dls_bypass_v6 && meoi.meoi_l3proto == ETHERTYPE_IPV6)) { DTRACE_PROBE4(rx__fanout, mblk_t *, mp, mac_ether_offload_info_t *, &meoi, mac_soft_ring_set_t *, mac_srs, pkt_type_t, OTH); - FANOUT_ENQUEUE_MP(headmp[OTH], tailmp[OTH], - cnt[OTH], bw_ctl, sz[OTH], sz1, mp); + FANOUT_ENQUEUE_MP(headmp[OTH], tailmp[OTH], cnt[OTH], + sz[OTH], sz1, mp); continue; } @@ -1659,8 +1660,8 @@ mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) DTRACE_PROBE4(rx__fanout, mblk_t *, mp, mac_ether_offload_info_t *, &meoi, mac_soft_ring_set_t *, mac_srs, pkt_type_t, OTH); - FANOUT_ENQUEUE_MP(headmp[OTH], tailmp[OTH], - cnt[OTH], bw_ctl, sz[OTH], sz1, mp); + FANOUT_ENQUEUE_MP(headmp[OTH], tailmp[OTH], cnt[OTH], + sz[OTH], sz1, mp); continue; } @@ -1690,7 +1691,7 @@ mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) mac_ether_offload_info_t *, &meoi, mac_soft_ring_set_t *, mac_srs, pkt_type_t, type); FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type], - bw_ctl, sz[type], sz1, mp); + sz[type], sz1, mp); } for (pkt_type_t type = V4_TCP; type < UNDEF; type++) { @@ -1917,15 +1918,14 @@ src_dst_based_fanout: static void mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) { - mblk_t *headmp[MAX_SR_TYPES][MAX_SR_FANOUT]; - mblk_t *tailmp[MAX_SR_TYPES][MAX_SR_FANOUT]; - int cnt[MAX_SR_TYPES][MAX_SR_FANOUT]; - size_t sz[MAX_SR_TYPES][MAX_SR_FANOUT]; - mac_client_impl_t *mcip = mac_srs->srs_mcip; + mblk_t *headmp[ST_RING_NUM_PROTO][MAX_SR_FANOUT]; + mblk_t *tailmp[ST_RING_NUM_PROTO][MAX_SR_FANOUT]; + int cnt[ST_RING_NUM_PROTO][MAX_SR_FANOUT]; + size_t sz[ST_RING_NUM_PROTO][MAX_SR_FANOUT]; + mac_client_impl_t *mcip = mac_srs->srs_mcip; const boolean_t is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER); - const boolean_t bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0); /* * If we don't have a Rx ring, S/W classification would have done @@ -1938,14 +1938,17 @@ mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER; /* - * Some clients, such as non Ethernet, need DLS processing in - * the Rx path. Such clients clear the SRST_DLS_BYPASS flag. - * DLS bypass may also be disabled via the - * MCIS_RX_BYPASS_DISABLE flag, but this is only consumed by - * sun4v vsw currently. + * Some clients, such as non Ethernet, need DLS processing in the Rx + * path. Such clients clear the bypass flag. DLS bypass may also be + * disabled via the MCIS_RX_BYPASS_DISABLE flag, but this is only + * consumed by sun4v vsw currently. */ - const boolean_t dls_bypass = - ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) && + const boolean_t dls_bypass_v4 = + ((mac_srs->srs_type & SRST_DLS_BYPASS_V4) != 0) && + ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0); + + const boolean_t dls_bypass_v6 = + ((mac_srs->srs_type & SRST_DLS_BYPASS_V6) != 0) && ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0); /* @@ -1958,10 +1961,10 @@ mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) */ const int fanout_cnt = mac_srs->srs_tcp_ring_count; - bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *)); - bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *)); - bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int)); - bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t)); + bzero(headmp, sizeof (headmp)); + bzero(tailmp, sizeof (tailmp)); + bzero(cnt, sizeof (cnt)); + bzero(sz, sizeof (sz)); /* * We got a chain from SRS that we need to send to the soft rings. @@ -2034,7 +2037,8 @@ mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) dstaddr = non_ether_mhi.mhi_daddr; } - if (!dls_bypass) { + if ((!dls_bypass_v4 && meoi.meoi_l3proto == ETHERTYPE_IP) || + (!dls_bypass_v6 && meoi.meoi_l3proto == ETHERTYPE_IPV6)) { if (mac_rx_srs_long_fanout(mac_srs, mp, meoi.meoi_l3proto, meoi.meoi_l2hlen, &type, &indx) == -1) { @@ -2046,8 +2050,7 @@ mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) mac_ether_offload_info_t *, &meoi, mac_soft_ring_set_t *, mac_srs, pkt_type_t, type); FANOUT_ENQUEUE_MP(headmp[type][indx], - tailmp[type][indx], - cnt[type][indx], bw_ctl, + tailmp[type][indx], cnt[type][indx], sz[type][indx], sz1, mp); continue; } @@ -2149,7 +2152,7 @@ mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) mac_ether_offload_info_t *, &meoi, mac_soft_ring_set_t *, mac_srs, pkt_type_t, type); FANOUT_ENQUEUE_MP(headmp[type][indx], - tailmp[type][indx], cnt[type][indx], bw_ctl, + tailmp[type][indx], cnt[type][indx], sz[type][indx], sz1, mp); continue; } @@ -2224,7 +2227,7 @@ mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head) mac_ether_offload_info_t *, &meoi, mac_soft_ring_set_t *, mac_srs, pkt_type_t, type); FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx], - cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp); + cnt[type][indx], sz[type][indx], sz1, mp); } for (pkt_type_t type = V4_TCP; type < UNDEF; type++) { @@ -4678,9 +4681,7 @@ mac_tx_notify(mac_impl_t *mip) (ringp)->s_ring_last = (tail); \ (ringp)->s_ring_count += (cnt); \ ASSERT((ringp)->s_ring_count > 0); \ - if ((ringp)->s_ring_type & ST_RING_BW_CTL) { \ - (ringp)->s_ring_size += sz; \ - } \ + (ringp)->s_ring_size += sz; \ } /* diff --git a/usr/src/uts/common/io/mac/mac_soft_ring.c b/usr/src/uts/common/io/mac/mac_soft_ring.c index 100b46d9aa..d82abff249 100644 --- a/usr/src/uts/common/io/mac/mac_soft_ring.c +++ b/usr/src/uts/common/io/mac/mac_soft_ring.c @@ -22,7 +22,7 @@ * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2018 Joyent, Inc. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* @@ -597,30 +597,43 @@ mac_soft_ring_poll(mac_soft_ring_t *ringp, size_t bytes_to_pickup) } /* - * mac_soft_ring_dls_bypass - * * Enable direct client (IP) callback function from the softrings. * Callers need to make sure they don't need any DLS layer processing */ void -mac_soft_ring_dls_bypass(void *arg, mac_direct_rx_t rx_func, void *rx_arg1) +mac_soft_ring_dls_bypass_enable(mac_soft_ring_t *softring, + mac_direct_rx_t rx_func, void *rx_arg1) { - mac_soft_ring_t *softring = arg; - mac_soft_ring_set_t *srs; - VERIFY3P(rx_func, !=, NULL); - mutex_enter(&softring->s_ring_lock); softring->s_ring_rx_func = rx_func; softring->s_ring_rx_arg1 = rx_arg1; mutex_exit(&softring->s_ring_lock); +} - srs = softring->s_ring_set; - mutex_enter(&srs->srs_lock); - srs->srs_type |= SRST_DLS_BYPASS; - mutex_exit(&srs->srs_lock); +/* Disable DLS bypass. */ +void +mac_soft_ring_dls_bypass_disable(mac_soft_ring_t *softring, + mac_client_impl_t *mcip) +{ + mutex_enter(&softring->s_ring_lock); + /* + * Before modifying the ring state we first wait for any in-progress + * processing to stop. + */ + while (softring->s_ring_state & S_RING_PROC) { + softring->s_ring_state |= S_RING_CLIENT_WAIT; + cv_wait(&softring->s_ring_client_cv, + &softring->s_ring_lock); + } + + softring->s_ring_state &= ~S_RING_CLIENT_WAIT; + softring->s_ring_rx_func = mac_rx_deliver; + softring->s_ring_rx_arg1 = mcip; + mutex_exit(&softring->s_ring_lock); } + /* * mac_soft_ring_signal * diff --git a/usr/src/uts/common/io/mac/mac_stat.c b/usr/src/uts/common/io/mac/mac_stat.c index 7fc73b8abf..5114f59dff 100644 --- a/usr/src/uts/common/io/mac/mac_stat.c +++ b/usr/src/uts/common/io/mac/mac_stat.c @@ -22,7 +22,7 @@ * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2018 Joyent, Inc. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* @@ -1113,16 +1113,21 @@ mac_soft_ring_stat_create(mac_soft_ring_t *ringp) return; is_tx_srs = ((mac_srs->srs_type & SRST_TX) != 0); - if (is_tx_srs) { /* tx side hardware lane */ + if (is_tx_srs) { ASSERT(ring != NULL); (void) snprintf(statname, sizeof (statname), "mac_tx_hwlane%d", ring->mr_index); i_mac_tx_hwlane_stat_create(ringp, flent->fe_flow_name, statname); - } else { /* rx side fanout */ - /* Maintain single stat for (tcp, udp, oth) */ + } else { + /* + * We maintain a single "fanout lane" stat per set of rx + * protocol softrings. That is, each set of (TCP/TCP6, UDP/UDP6, + * OTH) softrings counts as a single lane. + */ if (ringp->s_ring_type & ST_RING_TCP) { int index; + int fanout_lane; mac_soft_ring_t *softring; for (index = 0, softring = mac_srs->srs_soft_ring_head; @@ -1132,13 +1137,15 @@ mac_soft_ring_stat_create(mac_soft_ring_t *ringp) break; } + fanout_lane = index / ST_RING_NUM_PROTO; + if (mac_srs->srs_ring == NULL) { (void) snprintf(statname, sizeof (statname), - "mac_rx_swlane0_fanout%d", index/3); + "mac_rx_swlane0_fanout%d", fanout_lane); } else { (void) snprintf(statname, sizeof (statname), "mac_rx_hwlane%d_fanout%d", - mac_srs->srs_ring->mr_index, index/3); + mac_srs->srs_ring->mr_index, fanout_lane); } i_mac_rx_fanout_stat_create(ringp, flent->fe_flow_name, statname); diff --git a/usr/src/uts/common/sys/mac.h b/usr/src/uts/common/sys/mac.h index d79d625f4d..ba3cd53da8 100644 --- a/usr/src/uts/common/sys/mac.h +++ b/usr/src/uts/common/sys/mac.h @@ -24,6 +24,7 @@ * Copyright 2018 Joyent, Inc. * Copyright (c) 2015 Garrett D'Amore * Copyright 2020 RackTop Systems, Inc. + * Copyright 2026 Oxide Computer Company */ #ifndef _SYS_MAC_H @@ -497,10 +498,15 @@ typedef int (*mac_resource_bind_t)(void *, typedef void (*mac_resource_remove_t)(void *, void *); typedef void (*mac_resource_quiesce_t)(void *, void *); typedef void (*mac_resource_restart_t)(void *, void *); -typedef int (*mac_resource_modify_t)(void *, void *, - mac_resource_t *); -typedef void (*mac_change_upcall_t)(void *, mac_direct_rx_t, - void *); + +typedef struct mac_resource_cb_s { + mac_resource_add_t mrc_add; + mac_resource_remove_t mrc_remove; + mac_resource_quiesce_t mrc_quiesce; + mac_resource_restart_t mrc_restart; + mac_resource_bind_t mrc_bind; + void *mrc_arg; +} mac_resource_cb_t; /* * MAC-Type plugin interfaces diff --git a/usr/src/uts/common/sys/mac_client_impl.h b/usr/src/uts/common/sys/mac_client_impl.h index 4702f00bbb..2d04e11a0a 100644 --- a/usr/src/uts/common/sys/mac_client_impl.h +++ b/usr/src/uts/common/sys/mac_client_impl.h @@ -22,7 +22,7 @@ * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright (c) 2012, Joyent, Inc. All rights reserved. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* * Copyright 2018 Joyent, Inc. @@ -150,14 +150,18 @@ struct mac_client_impl_s { /* Protected by */ uint_t mci_nvids; /* mci_rw_lock */ volatile uint32_t mci_vidcache; /* VID cache */ - /* Resource Management Functions */ - mac_resource_add_t mci_resource_add; /* SL */ - mac_resource_remove_t mci_resource_remove; /* SL */ - mac_resource_quiesce_t mci_resource_quiesce; /* SL */ - mac_resource_restart_t mci_resource_restart; /* SL */ - mac_resource_bind_t mci_resource_bind; /* SL */ - void *mci_resource_arg; /* SL */ - + /* + * Resource Management Callback Functions + * + * A mac client may have both an IPv4 and IPv6 ill_t active on it. In + * order to avoid stomping on each other we give each their own resource + * callbacks. At this time resources are used solely by TCP softrings + * for the purpose of IP ring/squeue creation and polling. Currently the + * callbacks are identical across protocol types, save the mrc_arg, + * which is used to pass the ill_t up to IP. + */ + mac_resource_cb_t mci_rcb4; /* SL */ + mac_resource_cb_t mci_rcb6; /* SL */ /* Tx notify callback */ kmutex_t mci_tx_cb_lock; diff --git a/usr/src/uts/common/sys/mac_client_priv.h b/usr/src/uts/common/sys/mac_client_priv.h index 701fb719b6..63b3cfc33c 100644 --- a/usr/src/uts/common/sys/mac_client_priv.h +++ b/usr/src/uts/common/sys/mac_client_priv.h @@ -23,7 +23,7 @@ * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2018 Joyent, Inc. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ /* @@ -64,7 +64,9 @@ extern void mac_stop(mac_handle_t); extern void mac_ioctl(mac_handle_t, queue_t *, mblk_t *); extern link_state_t mac_link_get(mac_handle_t); -extern void mac_resource_set(mac_client_handle_t, mac_resource_add_t, void *); +extern void mac_resource_set(mac_client_handle_t, mac_resource_cb_t *, + boolean_t); +extern void mac_resource_clear(mac_client_handle_t, boolean_t); extern dev_info_t *mac_devinfo_get(mac_handle_t); extern void *mac_driver(mac_handle_t); extern boolean_t mac_capab_get(mac_handle_t, mac_capab_t, void *); @@ -76,10 +78,6 @@ extern int mac_vlan_header_info(mac_handle_t, mblk_t *, mac_header_info_t *); extern mblk_t *mac_header_cook(mac_handle_t, mblk_t *); extern mblk_t *mac_header_uncook(mac_handle_t, mblk_t *); -extern void mac_resource_set_common(mac_client_handle_t, - mac_resource_add_t, mac_resource_remove_t, mac_resource_quiesce_t, - mac_resource_restart_t, mac_resource_bind_t, void *); - extern void mac_perim_enter_by_mh(mac_handle_t, mac_perim_handle_t *); extern int mac_perim_enter_by_macname(const char *, mac_perim_handle_t *); extern int mac_perim_enter_by_linkid(datalink_id_t, mac_perim_handle_t *); @@ -90,8 +88,8 @@ extern uint16_t mac_client_vid(mac_client_handle_t); extern int mac_vnic_unicast_set(mac_client_handle_t, const uint8_t *); extern boolean_t mac_client_is_vlan_vnic(mac_client_handle_t); -extern void mac_client_poll_enable(mac_client_handle_t); -extern void mac_client_poll_disable(mac_client_handle_t); +extern void mac_client_poll_enable(mac_client_handle_t, boolean_t); +extern void mac_client_poll_disable(mac_client_handle_t, boolean_t); /* * Flow-related APIs for MAC clients. diff --git a/usr/src/uts/common/sys/mac_soft_ring.h b/usr/src/uts/common/sys/mac_soft_ring.h index 1b2d7687a5..d93dedfffc 100644 --- a/usr/src/uts/common/sys/mac_soft_ring.h +++ b/usr/src/uts/common/sys/mac_soft_ring.h @@ -23,7 +23,7 @@ * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2017 Joyent, Inc. - * Copyright 2025 Oxide Computer Company + * Copyright 2026 Oxide Computer Company */ #ifndef _SYS_MAC_SOFT_RING_H @@ -108,9 +108,9 @@ struct mac_soft_ring_s { timeout_id_t s_ring_tid; /* timer id of pending timeout() */ kthread_t *s_ring_worker; /* kernel thread id */ char s_ring_name[S_RING_NAMELEN + 1]; - uint32_t s_ring_total_inpkt; - uint32_t s_ring_total_rbytes; - uint32_t s_ring_drops; + uint64_t s_ring_total_inpkt; + uint64_t s_ring_total_rbytes; + uint64_t s_ring_drops; struct mac_client_impl_s *s_ring_mcip; kstat_t *s_ring_ksp; @@ -396,13 +396,16 @@ struct mac_soft_ring_set_s { #define ST_RING_TCP 0x0004 #define ST_RING_UDP 0x0008 #define ST_RING_OTH 0x0010 - -#define ST_RING_BW_CTL 0x0020 #define ST_RING_TX 0x0040 #define ST_RING_TCP6 0x0080 #define ST_RING_UDP6 0x0100 +/* + * The total number of softring protocol lanes: TCP, TCP6, UDP, UDP6, OTH. + */ +#define ST_RING_NUM_PROTO 5 + /* * State flags. */ @@ -428,11 +431,6 @@ struct mac_soft_ring_set_s { */ #define S_RING_BIND_NONE -1 -/* - * defines for srs_type - identifies a link or a sub-flow - * and other static characteristics of a SRS like a tx - * srs, tcp only srs, etc. - */ #define SRST_LINK 0x00000001 #define SRST_FLOW 0x00000002 #define SRST_NO_SOFT_RINGS 0x00000004 @@ -447,8 +445,10 @@ struct mac_soft_ring_set_s { #define SRST_BW_CONTROL 0x00000200 #define SRST_DIRECT_POLL 0x00000400 -#define SRST_DLS_BYPASS 0x00001000 -#define SRST_CLIENT_POLL_ENABLED 0x00002000 +#define SRST_DLS_BYPASS_V4 0x00001000 +#define SRST_DLS_BYPASS_V6 0x00002000 +#define SRST_CLIENT_POLL_V4 0x00004000 +#define SRST_CLIENT_POLL_V6 0x00008000 /* * soft ring set flags. These bits are dynamic in nature and get @@ -649,7 +649,10 @@ extern void mac_soft_ring_worker_wakeup(mac_soft_ring_t *); extern void mac_soft_ring_blank(void *, time_t, uint_t, int); extern mblk_t *mac_soft_ring_poll(mac_soft_ring_t *, size_t); extern void mac_soft_ring_destroy(mac_soft_ring_t *); -extern void mac_soft_ring_dls_bypass(void *, mac_direct_rx_t, void *); +extern void mac_soft_ring_dls_bypass_enable(mac_soft_ring_t *, mac_direct_rx_t, + void *); +extern void mac_soft_ring_dls_bypass_disable(mac_soft_ring_t *, + mac_client_impl_t *); /* Rx SRS */ extern mac_soft_ring_set_t *mac_srs_create(struct mac_client_impl_s *, @@ -663,9 +666,9 @@ extern void mac_tx_srs_retarget_intr(mac_soft_ring_set_t *); extern void mac_srs_quiesce_initiate(mac_soft_ring_set_t *); extern void mac_srs_client_poll_enable(struct mac_client_impl_s *, - mac_soft_ring_set_t *); + mac_soft_ring_set_t *, boolean_t); extern void mac_srs_client_poll_disable(struct mac_client_impl_s *, - mac_soft_ring_set_t *); + mac_soft_ring_set_t *, boolean_t); extern void mac_srs_client_poll_quiesce(struct mac_client_impl_s *, mac_soft_ring_set_t *); extern void mac_srs_client_poll_restart(struct mac_client_impl_s *, diff --git a/usr/src/uts/intel/ip/ip.global-objs.debug64 b/usr/src/uts/intel/ip/ip.global-objs.debug64 index 691b7da537..9c42a6c47c 100644 --- a/usr/src/uts/intel/ip/ip.global-objs.debug64 +++ b/usr/src/uts/intel/ip/ip.global-objs.debug64 @@ -22,6 +22,7 @@ # Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. # Copyright 2011 Nexenta Systems, Inc. All rights reserved # Copyright 2019 Joyent, Inc. All rights reserved +# Copyright 2026 Oxide Computer Company # arp_m_tbl @@ -256,6 +257,7 @@ squeue_drain_ms squeue_drain_ns squeue_drain_stack_needed squeue_drain_stack_toodeep +squeue_poll_budget_bytes tcp_acceptor_rinit tcp_acceptor_winit tcp_conn_cache diff --git a/usr/src/uts/intel/ip/ip.global-objs.obj64 b/usr/src/uts/intel/ip/ip.global-objs.obj64 index 624f9984f0..60c2d5c86a 100644 --- a/usr/src/uts/intel/ip/ip.global-objs.obj64 +++ b/usr/src/uts/intel/ip/ip.global-objs.obj64 @@ -22,6 +22,7 @@ # Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. # Copyright 2011 Nexenta Systems, Inc. All rights reserved # Copyright 2019 Joyent, Inc. All rights reserved +# Copyright 2026 Oxide Computer Company # arp_m_tbl @@ -253,6 +254,7 @@ squeue_drain_ms squeue_drain_ns squeue_drain_stack_needed squeue_drain_stack_toodeep +squeue_poll_budget_bytes tcp_acceptor_rinit tcp_acceptor_winit tcp_conn_cache -- 2.51.2