gems-kernel/source/THIRDPARTY/xnu/bsd/skywalk/channel/os_channel.h

/*
 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _SKYWALK_OS_CHANNEL_H_
#define _SKYWALK_OS_CHANNEL_H_

#ifdef PRIVATE

#include <stdint.h>
#include <sys/types.h>
#include <sys/cdefs.h>
#include <uuid/uuid.h>
#include <mach/vm_types.h>
#include <skywalk/os_nexus.h>
#include <skywalk/os_packet.h>
#ifndef KERNEL
#include <skywalk/os_channel_event.h>
#include <net/if_var.h>
#endif /* !KERNEL */

/*
 * Compiler guards used by Libnetcore.
 */
#define OS_CHANNEL_HAS_NUM_BUFFERS_ATTR 1 /* CHANNEL_ATTR_NUM_BUFFERS */
#define OS_CHANNEL_HAS_LARGE_PACKET 1     /* CHANNEL_ATTR_LARGE_BUF_SIZE and */
                                          /* os_channel_large_packet_alloc() */

/* Flow advisory table index */
typedef uint32_t flowadv_idx_t;
#define FLOWADV_IDX_NONE                ((flowadv_idx_t)-1)

/*
 * Channel ring direction.
 */
typedef enum {
	CHANNEL_DIR_TX_RX,      /* default: TX and RX ring(s) */
	CHANNEL_DIR_TX,         /* (monitor) only TX ring(s) */
	CHANNEL_DIR_RX          /* (monitor) only RX ring(s) */
} ring_dir_t;

/*
 * Channel ring ID.
 */
typedef uint32_t ring_id_t;
#define CHANNEL_RING_ID_ANY             ((ring_id_t)-1)

typedef enum {
	CHANNEL_FIRST_TX_RING,
	CHANNEL_LAST_TX_RING,
	CHANNEL_FIRST_RX_RING,
	CHANNEL_LAST_RX_RING
} ring_id_type_t;

/* Sync mode values */
typedef enum {
	CHANNEL_SYNC_TX,        /* synchronize TX ring(s) */
	CHANNEL_SYNC_RX,        /* synchronize RX ring(s) */
#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
	CHANNEL_SYNC_UPP        /* synchronize packet pool rings only */
#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */
} sync_mode_t;

/* Sync flags */
typedef uint32_t sync_flags_t;
#if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE)
#define CHANNEL_SYNCF_ALLOC        0x1     /* synchronize alloc ring */
#define CHANNEL_SYNCF_FREE         0x2     /* synchronize free ring */
#define CHANNEL_SYNCF_PURGE        0x4     /* purge user packet pool */
#define CHANNEL_SYNCF_ALLOC_BUF    0x8     /* synchronize buflet alloc ring */
#define CHANNEL_SYNCF_LARGE_ALLOC  0x10    /* synchronize large alloc ring */
#endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */

/*
 * Opaque handles.
 */
struct channel;
struct channel_ring_desc;
struct __slot_desc;
struct channel_attr;

typedef struct channel                  *channel_t;
typedef struct channel_ring_desc        *channel_ring_t;
typedef struct __slot_desc              *channel_slot_t;
typedef struct channel_attr             *channel_attr_t;

/*
 * Channel monitor types.
 */
typedef enum {
	CHANNEL_MONITOR_OFF,            /* default */
	CHANNEL_MONITOR_NO_COPY,        /* zero-copy (delayed) mode */
	CHANNEL_MONITOR_COPY            /* copy (immediate) mode */
} channel_monitor_type_t;

/*
 * Channel threshold unit types.
 */
typedef enum {
	CHANNEL_THRESHOLD_UNIT_SLOTS,   /* unit in slots (default) */
	CHANNEL_THRESHOLD_UNIT_BYTES,   /* unit in bytes */
} channel_threshold_unit_t;

/*
 * Channel attribute types gettable/settable via os_channel_attr_{get,set}.
 *
 *     g: retrievable at any time
 *     s: settable at any time
 *     S: settable once, only at creation time
 */
typedef enum {
	CHANNEL_ATTR_TX_RINGS,          /* (g) # of transmit rings */
	CHANNEL_ATTR_RX_RINGS,          /* (g) # of receive rings */
	CHANNEL_ATTR_TX_SLOTS,          /* (g) # of slots per transmit ring */
	CHANNEL_ATTR_RX_SLOTS,          /* (g) # of slots per receive ring */
	CHANNEL_ATTR_SLOT_BUF_SIZE,     /* (g) buffer per slot (bytes) */
	CHANNEL_ATTR_SLOT_META_SIZE,    /* (g) metadata per slot (bytes) */
	CHANNEL_ATTR_EXCLUSIVE,         /* (g/s) bool: exclusive open */
	CHANNEL_ATTR_NO_AUTO_SYNC,      /* (g/s) bool: will do explicit sync */
	CHANNEL_ATTR_MONITOR,           /* (g/s) see channel_monitor_type_t */
	CHANNEL_ATTR_TX_LOWAT_UNIT,     /* (g/s) see channel_threshold_unit_t */
	CHANNEL_ATTR_TX_LOWAT_VALUE,    /* (g/s) transmit low-watermark */
	CHANNEL_ATTR_RX_LOWAT_UNIT,     /* (g/s) see channel_threshold_unit_t */
	CHANNEL_ATTR_RX_LOWAT_VALUE,    /* (g/s) receive low-watermark */
	CHANNEL_ATTR_NEXUS_TYPE,        /* (g) nexus type */
	CHANNEL_ATTR_NEXUS_EXTENSIONS,  /* (g) nexus extension(s) */
	CHANNEL_ATTR_NEXUS_MHINTS,      /* (g) nexus memory usage hints */
	CHANNEL_ATTR_TX_HOST_RINGS,     /* (g) # of transmit host rings */
	CHANNEL_ATTR_RX_HOST_RINGS,     /* (g) # of receive host rings */
	CHANNEL_ATTR_NEXUS_IFINDEX,     /* (g) nexus network interface index */
	CHANNEL_ATTR_NEXUS_STATS_SIZE,  /* (g) nexus statistics region size */
	CHANNEL_ATTR_NEXUS_FLOWADV_MAX, /* (g) # of flow advisory entries */
	CHANNEL_ATTR_NEXUS_META_TYPE,   /* (g) nexus metadata type */
	CHANNEL_ATTR_NEXUS_META_SUBTYPE, /* (g) nexus metadata subtype */
	CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD, /* (g) nexus checksum offload */
	CHANNEL_ATTR_USER_PACKET_POOL,  /* (g/s) bool: use user packet pool */
	CHANNEL_ATTR_NEXUS_ADV_SIZE,    /* (g) nexus advisory region size */
	CHANNEL_ATTR_NEXUS_DEFUNCT_OK,  /* (g/s) bool: allow defunct */
	CHANNEL_ATTR_FILTER,            /* (g/s) bool: filter mode */
	CHANNEL_ATTR_EVENT_RING,        /* (g/s) bool: enable event ring */
	CHANNEL_ATTR_MAX_FRAGS,         /* (g) max length of buflet chain */
	CHANNEL_ATTR_NUM_BUFFERS,       /* (g) # of buffers in user pool */
	CHANNEL_ATTR_LOW_LATENCY,       /* (g/s) bool: low latency channel */
	CHANNEL_ATTR_LARGE_BUF_SIZE,    /* (g) large buffer size (bytes) */
} channel_attr_type_t;

/*
 * Channel nexus metadata type.
 */
typedef enum {
	CHANNEL_NEXUS_META_TYPE_INVALID = 0,
	CHANNEL_NEXUS_META_TYPE_QUANTUM, /* OK for os_packet quantum APIs */
	CHANNEL_NEXUS_META_TYPE_PACKET,  /* OK for all os_packet APIs */
} channel_nexus_meta_type_t;

/*
 * Channel nexus metadata subtype.
 */
typedef enum {
	CHANNEL_NEXUS_META_SUBTYPE_INVALID = 0,
	CHANNEL_NEXUS_META_SUBTYPE_PAYLOAD,
	CHANNEL_NEXUS_META_SUBTYPE_RAW,
} channel_nexus_meta_subtype_t;

/*
 * Valid values for CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD
 */
#define CHANNEL_NEXUS_CHECKSUM_PARTIAL  0x1     /* partial checksum */

/*
 * Channel statistics ID.
 */
typedef enum {
	CHANNEL_STATS_ID_IP = 0,        /* struct ip_stats */
	CHANNEL_STATS_ID_IP6,           /* struct ip6_stats */
	CHANNEL_STATS_ID_TCP,           /* struct tcp_stats */
	CHANNEL_STATS_ID_UDP,           /* struct udp_stats */
	CHANNEL_STATS_ID_QUIC,          /* struct quic_stats */
} channel_stats_id_t;

/*
 * Slot properties.  Structure is aligned to allow for efficient copy.
 *
 * Fields except for sp_{flags,len} are immutables (I).  The system will
 * verify for correctness during os_channel_put() across the immutable
 * fields, and will abort the process if it detects inconsistencies.
 * This is meant to help with debugging, since it indicates bugs and/or
 * memory corruption.
 */
typedef struct slot_prop {
	uint16_t sp_flags;              /* private flags */
	uint16_t sp_len;                /* length for this slot */
	uint32_t sp_idx;                /* (I) slot index */
	mach_vm_address_t sp_ext_ptr;   /* (I) pointer for indirect buffer */
	mach_vm_address_t sp_buf_ptr;   /* (I) pointer for native buffer */
	mach_vm_address_t sp_mdata_ptr; /* (I) pointer for metadata */
	uint32_t _sp_pad[8];            /* reserved */
} slot_prop_t __attribute__((aligned(sizeof(uint64_t))));

#ifndef KERNEL
/*
 * User APIs.
 */
#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
__BEGIN_DECLS
/*
 * Creates a Channel attribute object.
 *
 * This must be paired with a os_channel_attr_destroy() on the handle.
 */
extern channel_attr_t os_channel_attr_create(void);

/*
 * Clones a Channel attribute object.  If source attribute is NULL
 * it behaves just like os_channel_attr_create();
 *
 * This must be paired with a os_channel_attr_destroy() on the handle.
 */
extern channel_attr_t os_channel_attr_clone(const channel_attr_t);

/*
 * Sets a value for a given attribute type on a Channel attribute object.
 */
extern int os_channel_attr_set(const channel_attr_t attr,
    const channel_attr_type_t type, const uint64_t value);

/*
 * Sets a key blob on a Channel attribute object.  Existing key blob
 * information in the attribute object will be removed, if any, and
 * replaced with the new key blob.  Specifying 0 for key_len will
 * clear the key stored in the attribute object.  The maximum key
 * length is specified by NEXUS_MAX_KEY_LEN.
 */
extern int os_channel_attr_set_key(const channel_attr_t attr,
    const void *key, const uint32_t key_len);

/*
 * Gets a value for a given attribute type on a Channel attribute object.
 */
extern int os_channel_attr_get(const channel_attr_t attr,
    const channel_attr_type_t type, uint64_t *value);

/*
 * Gets a key blob on a Channel attribute object.  If key is NULL,
 * returns the length of the key blob with key_len, so caller knows
 * how much to allocate space for key blob.
 */
extern int os_channel_attr_get_key(const channel_attr_t attr,
    void *key, uint32_t *key_len);

/*
 * Destroys a Channel attribute object, along with all resources
 * associated with it (e.g. key blob).
 */
extern void os_channel_attr_destroy(const channel_attr_t attr);

/*
 * Opens a Channel to a Nexus provider instance.  Upon success, maps memory
 * region and allocates resources.
 *
 * This must be paired with a os_channel_destroy() on the handle, in order to
 * unmap the memory region and free resources.
 */
extern channel_t os_channel_create(const uuid_t uuid, const nexus_port_t port);

/*
 * Extended version of os_channel_create().
 */
extern channel_t os_channel_create_extended(const uuid_t uuid,
    const nexus_port_t port, const ring_dir_t dir, const ring_id_t rid,
    const channel_attr_t attr);

/*
 * Retrieves the file descriptor associated with the Channel.
 */
extern int os_channel_get_fd(const channel_t channel);

/*
 * Retrieves current channel attributes into the channel_attr_t handle.
 */
extern int os_channel_read_attr(const channel_t channel, channel_attr_t attr);

/*
 * Updates channel attributes based on those referred to by the channel_attr_t
 * handle.  See comments above on channel_attr_type_t; this routine will only
 * update attributes that are marked with 's' but not 'S'.
 */
extern int os_channel_write_attr(const channel_t channel, channel_attr_t attr);

/*
 * Retrieves channel's associated nexus type into *nexus_type, and the
 * provider-specific extension attribute into *ext.
 */
extern int os_channel_read_nexus_extension_info(const channel_t channel,
    nexus_type_t *nexus_type, uint64_t *ext);

/*
 * Non-blocking synchronization.  Channel handle may also be used
 * with kqueue(2), select(2) or poll(2) through the file descriptor.
 */
extern int os_channel_sync(const channel_t channel, const sync_mode_t mode);

/*
 * Destroys a Channel.
 */
extern void os_channel_destroy(const channel_t channel);

/*
 * Checks if a channel is defunct.  Returns non-zero if defunct.
 */
extern int os_channel_is_defunct(const channel_t channel);

/*
 * Data Movement APIs.
 *
 * Obtain channel_ring_t handle via os_channel_{tx,rx}_ring().  You will
 * need to specify the ring_id_t which identifies the ring — this is true
 * even for a single TX/RX ring case.  The Nexus provider can communicate
 * to the client the ID of the TX and RX ring that should be used to
 * communicate to it, through a contract between the two.  For instance,
 * it can tell the client to use first TX ring and first RX ring, etc.
 * through some side-channel.  It should not assume 0 or any other number
 * as ID, however, as the in-kernel Nexus object is the authoritative source
 * of truth.  This is where the os_channel_ring_id() call comes into the
 * picture, as it will return the first and last usable TX and RX ring IDs
 * for the Channel opened to that Nexus object.
 *
 * Once the TX or RX ring handle is obtained above, the client can ask for
 * the first usable slot in the ring through os_channel_get_next_slot()
 * passing NULL for the 'slot' parameter. This returns a channel_slot_t
 * handle that represents the slot, along with the properties of that slot
 * described by the slot_prop_t structure. If no slots are available, this
 * call returns a NULL handle.  It’s important to note that this
 * call does NOT advance the ring’s current slot pointer; calling this
 * multiple times in succession will yield the same result.
 *
 * The client proceeds to use the slot by examining the returned
 * slot_prop_t fields including the pointer to the internal buffer
 * associated with that slot.  Once the client is finished, it updates
 * the relevant slot_prop_t fields (e.g. length) and calls
 * os_channel_set_slot_properties() to apply them to the slot.
 *
 * To get the next slot, the client provides the non-NULL slot value obtained
 * from the previous call to os_channel_get_next_slot() as the 'slot' parameter
 * in its next invocation of that function.
 *
 * To advance the ring’s current pointer, the client invokes
 * os_channel_advance_slot() specifying the slot to advance past. If the slot
 * is invalid, this function returns a non-zero value.
 *
 * Once the client is ready to commit, call os_channel_sync() in
 * either/all directions.
 */
extern ring_id_t os_channel_ring_id(const channel_t channel,
    const ring_id_type_t type);
extern channel_ring_t os_channel_tx_ring(const channel_t channel,
    const ring_id_t rid);
extern channel_ring_t os_channel_rx_ring(const channel_t channel,
    const ring_id_t rid);
extern int os_channel_pending(const channel_ring_t ring);

/*
 * This returns a nexus-specific timestamp in nanoseconds taken at the
 * lasttime os_channel_sync() or its equivalent implicit kevent sync
 * was called
 */
extern uint64_t os_channel_ring_sync_time(const channel_ring_t ring);

/*
 * This returns a nexus-specific timestamp in nanoseconds to indicate
 * the time of last activity on the opposite end of the ring.
 * This is only updated when sync or kevent equivalent is called.
 */
extern uint64_t os_channel_ring_notify_time(const channel_ring_t ring);

/*
 * For TX ring os_channel_available_slot_count() returns the minimum number
 * of slots available availble for TX, and it is possible that
 * os_channel_get_next_slot() will return more slots than the what was
 * returned by an earlier call to os_channel_available_slot_count()
 */
extern uint32_t os_channel_available_slot_count(const channel_ring_t ring);
extern channel_slot_t os_channel_get_next_slot(const channel_ring_t ring,
    const channel_slot_t slot, slot_prop_t *prop);
extern int os_channel_advance_slot(channel_ring_t ring,
    const channel_slot_t slot);
extern void os_channel_set_slot_properties(const channel_ring_t ring,
    const channel_slot_t slot, const slot_prop_t *prop);

/*
 * Return the packet handle associated with a given slot of a ring.
 */
extern packet_t os_channel_slot_get_packet(const channel_ring_t ring,
    const channel_slot_t slot);

/*
 * Each nexus that the channel is connected to determines whether or
 * not there is a shareable statistics region identified by one of
 * the channel_stats_id_t values.  This routine returns a pointer to
 * such a region upon success, or NULL if not supported by the nexus.
 */
extern void *os_channel_get_stats_region(const channel_t channel,
    const channel_stats_id_t id);

/*
 * Each nexus that the channel is connected to determines whether or
 * not there is a nexus-wide advisory region.  This routine returns
 * a pointer to such a region upon success, or NULL if not supported
 * by the nexus.
 */
extern void *os_channel_get_advisory_region(const channel_t channel);

/*
 * Each nexus that supports flow admission control may be queried to
 * advise whether or not the channel is willing to admit more packets
 * for a given flow.  A return value of 0 indicates that the packets
 * for the flow are admissible.  If ENOBUFS is returned, the flow is
 * currently suspended, and further attempts to send more packets on
 * the ring may result in drops.  Any other error values indicate
 * that either the nexus doesn't support admission control, or the
 * arguments aren't valid.
 */
extern int os_channel_flow_admissible(const channel_ring_t ring,
    uuid_t flow_id, const flowadv_idx_t flow_index);

extern int os_channel_flow_adv_get_ce_count(const channel_ring_t chrd,
    uuid_t flow_id, const flowadv_idx_t flow_index, uint32_t *ce_cnt,
    uint32_t *pkt_cnt);
/*
 * Allocate a packet from the channel's packet pool.
 * Returns 0 on success with the packet handle in packet arg.
 * Note: os_channel_packet_alloc() & os_channel_packet_free() should be
 * serialized and should not be called from the different thread context.
 */
extern int
os_channel_packet_alloc(const channel_t chd, packet_t *packet);

/*
 * Allocate a large packet from the channel's packet pool.
 * Returns 0 on success with the packet handle in packet arg.
 * Note: os_channel_large_packet_alloc() & os_channel_packet_free() should be
 * serialized and should not be called from the different thread context.
 */
extern int
os_channel_large_packet_alloc(const channel_t chd, packet_t *packet);

/*
 * Free a packet allocated from the channel's packet pool.
 * Returns 0 on success
 * Note: os_channel_packet_alloc() & os_channel_packet_free() should be
 * serialized and should not be called from the different thread context.
 */
extern int
os_channel_packet_free(const channel_t chd, packet_t packet);

/*
 * Attach the given packet to a channel slot
 */
extern int
os_channel_slot_attach_packet(const channel_ring_t chrd,
    const channel_slot_t slot, packet_t packet);

/*
 * Detach a given packet from a channel slot
 */
extern int
os_channel_slot_detach_packet(const channel_ring_t chrd,
    const channel_slot_t slot, packet_t packet);

/*
 * purge packets from the channel's packet pool.
 * This API should be called at regular intervals by application to purge
 * unused packets from the channel's packet pool. Recommended interval is
 * 11 seconds.
 * Returns 0 on success.
 * Note: This call should be serialized with os_channel_packet_alloc() &
 * os_channel_packet_free() and should not be called from different
 * thread context.
 */
extern int
os_channel_packet_pool_purge(const channel_t chd);

/*
 * Retrieve handle to the next available event(s) on the channel.
 * os_event_get_next_event() can then called on the event handle to
 * retrieve the individual events from the handle.
 * Returns 0 on success, ENXIO if the channel is defunct.
 */
extern int
os_channel_get_next_event_handle(const channel_t chd,
    os_channel_event_handle_t *ehandle, os_channel_event_type_t *etype,
    uint32_t *nevents);

/*
 * Free an event retrieved from the channel.
 * Returns 0 on success, ENXIO if the channel is defunct.
 */
extern int
os_channel_event_free(const channel_t chd, os_channel_event_handle_t ehandle);

/*
 * API to retrieve the latest interface advisory report on the channel.
 * Returns 0 on succcess. If the return value is EAGAIN, caller can attempt
 * to retrieve the information again.
 */
extern int
os_channel_get_interface_advisory(const channel_t chd,
    struct ifnet_interface_advisory *advisory);

/*
 * API to configure interface advisory report on the channel.
 * Returns 0 on succcess.
 */
extern int
os_channel_configure_interface_advisory(const channel_t chd, boolean_t enable);

extern int
os_channel_buflet_alloc(const channel_t chd, buflet_t *bft);

extern int
os_channel_buflet_free(const channel_t chd, buflet_t ubft);
__END_DECLS
#endif  /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
#else /* KERNEL */
/*
 * Kernel APIs.
 */

/*
 * Opaque handles.
 */
struct kern_channel;
struct __kern_channel_ring;

typedef struct kern_channel             *kern_channel_t;
typedef struct __kern_channel_ring      *kern_channel_ring_t;
typedef struct __slot_desc              *kern_channel_slot_t;

/*
 * Slot properties (deprecated).
 */
struct kern_slot_prop {
	uint32_t _sp_pad[16];           /* reserved */
} __attribute__((aligned(sizeof(uint64_t))));

/*
 * @struct kern_channel_ring_stat_increment
 * @abstract Structure used to increment the per ring statistic counters.
 * @field kcrsi_slots_transferred  number of slots transferred
 * @filed kcrsi_bytes_transferred  number of bytes transferred
 */
struct kern_channel_ring_stat_increment {
	uint32_t        kcrsi_slots_transferred;
	uint32_t        kcrsi_bytes_transferred;
};

/*
 * Data Movement APIs.
 *
 * See block comment above for userland data movement APIs for general
 * concepts.  The main differences here are the kern_channel_notify()
 * and kern_channel_reclaim() calls that aren't available for userland.
 * These calls are typically invoked within the TX and RX sync callbacks
 * implemented by the nexus provider.
 *
 * For TX sync, kern_channel_reclaim() is normally called after the
 * provider has finished reclaiming slots that have been "transmitted".
 * In this case, this call is simply a way to indicate to the system
 * that such condition has happened.
 *
 * For RX sync, kern_channel_reclaim() must be called at the beginning
 * of the callback in order to reclaim user-released slots, and to
 * ensure that subsequent calls to kern_channel_available_slot_count()
 * or kern_channel_get_next_slot() operates on the most recent state.
 *
 * The kern_channel_notify() is used to post notifications to indicate
 * slot availability; this may result in the kernel event subsystem
 * posting readable and writable events.
 */
__BEGIN_DECLS
extern uint32_t kern_channel_notify(const kern_channel_ring_t, uint32_t flags);
extern uint32_t kern_channel_available_slot_count(
	const kern_channel_ring_t ring);
/*
 * NOTE: kern_channel_set_slot_properties(), kern_channel_get_next_slot(),
 * kern_channel_reclaim() and kern_channel_advance_slot() require that the
 * caller invokes them from within the sync callback context; they will
 * assert otherwise.
 */
extern void kern_channel_set_slot_properties(const kern_channel_ring_t,
    const kern_channel_slot_t slot, const struct kern_slot_prop *prop);
extern kern_channel_slot_t kern_channel_get_next_slot(
	const kern_channel_ring_t kring, const kern_channel_slot_t slot,
	struct kern_slot_prop *slot_prop);
extern uint32_t kern_channel_reclaim(const kern_channel_ring_t);
extern void kern_channel_advance_slot(const kern_channel_ring_t kring,
    kern_channel_slot_t slot);

/*
 * Packet.
 */
extern kern_packet_t kern_channel_slot_get_packet(
	const kern_channel_ring_t ring, const kern_channel_slot_t slot);

/*
 * NOTE: kern_channel_slot_attach_packet(), kern_channel_slot_detach_packet()
 * and kern_channel_ring_get_container() require that the caller invokes them
 * from within the sync callback context; they will assert otherwise.
 */
extern errno_t kern_channel_slot_attach_packet(const kern_channel_ring_t ring,
    const kern_channel_slot_t slot, kern_packet_t packet);
extern errno_t kern_channel_slot_detach_packet(const kern_channel_ring_t ring,
    const kern_channel_slot_t slot, kern_packet_t packet);
extern errno_t kern_channel_ring_get_container(const kern_channel_ring_t ring,
    kern_packet_t **array, uint32_t *count);
extern errno_t kern_channel_tx_refill(const kern_channel_ring_t ring,
    uint32_t pkt_limit, uint32_t byte_limit, boolean_t tx_doorbell_ctxt,
    boolean_t *pkts_pending);
extern errno_t kern_channel_get_service_class(const kern_channel_ring_t ring,
    kern_packet_svc_class_t *svc);
extern errno_t kern_netif_queue_get_service_class(kern_netif_queue_t,
    kern_packet_svc_class_t *);

/*
 * Misc.
 */
extern void *kern_channel_get_context(const kern_channel_t channel);
extern void *kern_channel_ring_get_context(const kern_channel_ring_t ring);
extern void *kern_channel_slot_get_context(const kern_channel_ring_t ring,
    const kern_channel_slot_t slot);

/*
 * NOTE: kern_channel_increment_ring_{net}_stats() requires
 * that the caller invokes it from within the sync callback context;
 * it will assert otherwise.
 */
extern void kern_channel_increment_ring_stats(kern_channel_ring_t ring,
    struct kern_channel_ring_stat_increment *stats);
extern void kern_channel_increment_ring_net_stats(kern_channel_ring_t ring,
    ifnet_t, struct kern_channel_ring_stat_increment *stats);

#ifdef BSD_KERNEL_PRIVATE
/* forward declare */
struct flowadv_fcentry;

/* Flow advisory token */
typedef uint32_t flowadv_token_t;

/*
 * Private, unexported KPIs.
 */
__private_extern__ errno_t kern_channel_slot_attach_packet_byidx(
	const kern_channel_ring_t kring, const uint32_t sidx, kern_packet_t ph);
__private_extern__ errno_t kern_channel_slot_detach_packet_byidx(
	const kern_channel_ring_t kring, const uint32_t sidx, kern_packet_t ph);
__private_extern__ void kern_channel_flowadv_clear(struct flowadv_fcentry *);
__private_extern__ void kern_channel_flowadv_report_ce_event(
	struct flowadv_fcentry *, uint32_t, uint32_t);
__private_extern__ void kern_channel_memstatus(struct proc *, uint32_t,
    struct kern_channel *);
__private_extern__ void kern_channel_defunct(struct proc *,
    struct kern_channel *);
__private_extern__ errno_t kern_channel_tx_refill_canblock(
	const kern_channel_ring_t, uint32_t, uint32_t, boolean_t, boolean_t *);
#endif /* BSD_KERNEL_PRIVATE */
__END_DECLS
#endif /* KERNEL */
#endif /* PRIVATE */
#endif /* !_SKYWALK_OS_CHANNEL_H_ */