3132 lines
86 KiB
C
3132 lines
86 KiB
C
|
/*
|
||
|
|
* Copyright (c) 2000-2019 Apple Computer, Inc. All rights reserved.
|
||
|
|
*
|
||
|
|
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
|
||
|
|
*
|
||
|
|
* This file contains Original Code and/or Modifications of Original Code
|
||
|
|
* as defined in and that are subject to the Apple Public Source License
|
||
|
|
* Version 2.0 (the 'License'). You may not use this file except in
|
||
|
|
* compliance with the License. The rights granted to you under the License
|
||
|
|
* may not be used to create, or enable the creation or redistribution of,
|
||
|
|
* unlawful or unlicensed copies of an Apple operating system, or to
|
||
|
|
* circumvent, violate, or enable the circumvention or violation of, any
|
||
|
|
* terms of an Apple operating system software license agreement.
|
||
|
|
*
|
||
|
|
* Please obtain a copy of the License at
|
||
|
|
* http://www.opensource.apple.com/apsl/ and read it before using this file.
|
||
|
|
*
|
||
|
|
* The Original Code and all software distributed under the License are
|
||
|
|
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
|
||
|
|
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
|
||
|
|
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
|
||
|
|
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
|
||
|
|
* Please see the License for the specific language governing rights and
|
||
|
|
* limitations under the License.
|
||
|
|
*
|
||
|
|
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
|
||
|
|
*/
|
||
|
|
/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
|
||
|
|
/*
|
||
|
|
* Copyright (c) 1989, 1993, 1995
|
||
|
|
* The Regents of the University of California. All rights reserved.
|
||
|
|
*
|
||
|
|
* Redistribution and use in source and binary forms, with or without
|
||
|
|
* modification, are permitted provided that the following conditions
|
||
|
|
* are met:
|
||
|
|
* 1. Redistributions of source code must retain the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer.
|
||
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer in the
|
||
|
|
* documentation and/or other materials provided with the distribution.
|
||
|
|
* 3. All advertising materials mentioning features or use of this software
|
||
|
|
* must display the following acknowledgement:
|
||
|
|
* This product includes software developed by the University of
|
||
|
|
* California, Berkeley and its contributors.
|
||
|
|
* 4. Neither the name of the University nor the names of its contributors
|
||
|
|
* may be used to endorse or promote products derived from this software
|
||
|
|
* without specific prior written permission.
|
||
|
|
*
|
||
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
||
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
||
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||
|
|
* SUCH DAMAGE.
|
||
|
|
*
|
||
|
|
* @(#)spec_vnops.c 8.14 (Berkeley) 5/21/95
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <sys/param.h>
|
||
|
|
#include <sys/proc_internal.h>
|
||
|
|
#include <sys/kauth.h>
|
||
|
|
#include <sys/systm.h>
|
||
|
|
#include <sys/kernel.h>
|
||
|
|
#include <sys/conf.h>
|
||
|
|
#include <sys/buf_internal.h>
|
||
|
|
#include <sys/mount_internal.h>
|
||
|
|
#include <sys/vnode_internal.h>
|
||
|
|
#include <sys/file_internal.h>
|
||
|
|
#include <sys/namei.h>
|
||
|
|
#include <sys/stat.h>
|
||
|
|
#include <sys/errno.h>
|
||
|
|
#include <sys/ioctl.h>
|
||
|
|
#include <sys/file.h>
|
||
|
|
#include <sys/user.h>
|
||
|
|
#include <sys/malloc.h>
|
||
|
|
#include <sys/disk.h>
|
||
|
|
#include <sys/uio_internal.h>
|
||
|
|
#include <sys/resource.h>
|
||
|
|
#include <machine/machine_routines.h>
|
||
|
|
#include <miscfs/specfs/specdev.h>
|
||
|
|
#include <vfs/vfs_support.h>
|
||
|
|
#include <vfs/vfs_disk_conditioner.h>
|
||
|
|
|
||
|
|
#include <kern/assert.h>
|
||
|
|
#include <kern/task.h>
|
||
|
|
#include <kern/sched_prim.h>
|
||
|
|
#include <kern/thread.h>
|
||
|
|
#include <kern/policy_internal.h>
|
||
|
|
#include <kern/timer_call.h>
|
||
|
|
#include <kern/waitq.h>
|
||
|
|
|
||
|
|
#include <pexpert/pexpert.h>
|
||
|
|
|
||
|
|
#include <sys/kdebug.h>
|
||
|
|
#include <libkern/section_keywords.h>
|
||
|
|
|
||
|
|
#if CONFIG_IO_COMPRESSION_STATS
|
||
|
|
#include <vfs/vfs_io_compression_stats.h>
|
||
|
|
#endif /* CONFIG_IO_COMPRESSION_STATS */
|
||
|
|
|
||
|
|
/* XXX following three prototypes should be in a header file somewhere */
|
||
|
|
extern dev_t chrtoblk(dev_t dev);
|
||
|
|
extern boolean_t iskmemdev(dev_t dev);
|
||
|
|
extern int bpfkqfilter(dev_t dev, struct knote *kn);
|
||
|
|
extern int ptsd_kqfilter(dev_t, struct knote *);
|
||
|
|
extern int ptmx_kqfilter(dev_t, struct knote *);
|
||
|
|
#if CONFIG_PHYS_WRITE_ACCT
|
||
|
|
uint64_t kernel_pm_writes; // to track the sync writes occurring during power management transitions
|
||
|
|
#endif /* CONFIG_PHYS_WRITE_ACCT */
|
||
|
|
|
||
|
|
|
||
|
|
struct vnode *speclisth[SPECHSZ];
|
||
|
|
|
||
|
|
/* symbolic sleep message strings for devices */
|
||
|
|
char devopn[] = "devopn";
|
||
|
|
char devio[] = "devio";
|
||
|
|
char devwait[] = "devwait";
|
||
|
|
char devin[] = "devin";
|
||
|
|
char devout[] = "devout";
|
||
|
|
char devioc[] = "devioc";
|
||
|
|
char devcls[] = "devcls";
|
||
|
|
|
||
|
|
#define VOPFUNC int (*)(void *)
|
||
|
|
|
||
|
|
int(**spec_vnodeop_p)(void *);
|
||
|
|
const struct vnodeopv_entry_desc spec_vnodeop_entries[] = {
|
||
|
|
{ .opve_op = &vnop_default_desc, .opve_impl = (VOPFUNC)(void (*)(void))vn_default_error },
|
||
|
|
{ .opve_op = &vnop_lookup_desc, .opve_impl = (VOPFUNC)spec_lookup }, /* lookup */
|
||
|
|
{ .opve_op = &vnop_create_desc, .opve_impl = (VOPFUNC)err_create }, /* create */
|
||
|
|
{ .opve_op = &vnop_mknod_desc, .opve_impl = (VOPFUNC)err_mknod }, /* mknod */
|
||
|
|
{ .opve_op = &vnop_open_desc, .opve_impl = (VOPFUNC)spec_open }, /* open */
|
||
|
|
{ .opve_op = &vnop_close_desc, .opve_impl = (VOPFUNC)spec_close }, /* close */
|
||
|
|
{ .opve_op = &vnop_access_desc, .opve_impl = (VOPFUNC)spec_access }, /* access */
|
||
|
|
{ .opve_op = &vnop_getattr_desc, .opve_impl = (VOPFUNC)spec_getattr }, /* getattr */
|
||
|
|
{ .opve_op = &vnop_setattr_desc, .opve_impl = (VOPFUNC)spec_setattr }, /* setattr */
|
||
|
|
{ .opve_op = &vnop_read_desc, .opve_impl = (VOPFUNC)spec_read }, /* read */
|
||
|
|
{ .opve_op = &vnop_write_desc, .opve_impl = (VOPFUNC)spec_write }, /* write */
|
||
|
|
{ .opve_op = &vnop_ioctl_desc, .opve_impl = (VOPFUNC)spec_ioctl }, /* ioctl */
|
||
|
|
{ .opve_op = &vnop_select_desc, .opve_impl = (VOPFUNC)spec_select }, /* select */
|
||
|
|
{ .opve_op = &vnop_revoke_desc, .opve_impl = (VOPFUNC)nop_revoke }, /* revoke */
|
||
|
|
{ .opve_op = &vnop_mmap_desc, .opve_impl = (VOPFUNC)err_mmap }, /* mmap */
|
||
|
|
{ .opve_op = &vnop_fsync_desc, .opve_impl = (VOPFUNC)spec_fsync }, /* fsync */
|
||
|
|
{ .opve_op = &vnop_remove_desc, .opve_impl = (VOPFUNC)err_remove }, /* remove */
|
||
|
|
{ .opve_op = &vnop_link_desc, .opve_impl = (VOPFUNC)err_link }, /* link */
|
||
|
|
{ .opve_op = &vnop_rename_desc, .opve_impl = (VOPFUNC)err_rename }, /* rename */
|
||
|
|
{ .opve_op = &vnop_mkdir_desc, .opve_impl = (VOPFUNC)err_mkdir }, /* mkdir */
|
||
|
|
{ .opve_op = &vnop_rmdir_desc, .opve_impl = (VOPFUNC)err_rmdir }, /* rmdir */
|
||
|
|
{ .opve_op = &vnop_symlink_desc, .opve_impl = (VOPFUNC)err_symlink }, /* symlink */
|
||
|
|
{ .opve_op = &vnop_readdir_desc, .opve_impl = (VOPFUNC)err_readdir }, /* readdir */
|
||
|
|
{ .opve_op = &vnop_readlink_desc, .opve_impl = (VOPFUNC)err_readlink }, /* readlink */
|
||
|
|
{ .opve_op = &vnop_inactive_desc, .opve_impl = (VOPFUNC)nop_inactive }, /* inactive */
|
||
|
|
{ .opve_op = &vnop_reclaim_desc, .opve_impl = (VOPFUNC)nop_reclaim }, /* reclaim */
|
||
|
|
{ .opve_op = &vnop_strategy_desc, .opve_impl = (VOPFUNC)spec_strategy }, /* strategy */
|
||
|
|
{ .opve_op = &vnop_pathconf_desc, .opve_impl = (VOPFUNC)spec_pathconf }, /* pathconf */
|
||
|
|
{ .opve_op = &vnop_advlock_desc, .opve_impl = (VOPFUNC)err_advlock }, /* advlock */
|
||
|
|
{ .opve_op = &vnop_bwrite_desc, .opve_impl = (VOPFUNC)spec_bwrite }, /* bwrite */
|
||
|
|
{ .opve_op = &vnop_pagein_desc, .opve_impl = (VOPFUNC)err_pagein }, /* Pagein */
|
||
|
|
{ .opve_op = &vnop_pageout_desc, .opve_impl = (VOPFUNC)err_pageout }, /* Pageout */
|
||
|
|
{ .opve_op = &vnop_copyfile_desc, .opve_impl = (VOPFUNC)err_copyfile }, /* Copyfile */
|
||
|
|
{ .opve_op = &vnop_blktooff_desc, .opve_impl = (VOPFUNC)spec_blktooff }, /* blktooff */
|
||
|
|
{ .opve_op = &vnop_offtoblk_desc, .opve_impl = (VOPFUNC)spec_offtoblk }, /* offtoblk */
|
||
|
|
{ .opve_op = &vnop_blockmap_desc, .opve_impl = (VOPFUNC)spec_blockmap }, /* blockmap */
|
||
|
|
{ .opve_op = (struct vnodeop_desc*)NULL, .opve_impl = (int (*)(void *))NULL }
|
||
|
|
};
|
||
|
|
const struct vnodeopv_desc spec_vnodeop_opv_desc =
|
||
|
|
{ .opv_desc_vector_p = &spec_vnodeop_p, .opv_desc_ops = spec_vnodeop_entries };
|
||
|
|
|
||
|
|
|
||
|
|
static void set_blocksize(vnode_t, dev_t);
|
||
|
|
|
||
|
|
#define LOWPRI_TIER1_WINDOW_MSECS 25
|
||
|
|
#define LOWPRI_TIER2_WINDOW_MSECS 100
|
||
|
|
#define LOWPRI_TIER3_WINDOW_MSECS 500
|
||
|
|
|
||
|
|
#define LOWPRI_TIER1_IO_PERIOD_MSECS 40
|
||
|
|
#define LOWPRI_TIER2_IO_PERIOD_MSECS 85
|
||
|
|
#define LOWPRI_TIER3_IO_PERIOD_MSECS 200
|
||
|
|
|
||
|
|
#define LOWPRI_TIER1_IO_PERIOD_SSD_MSECS 5
|
||
|
|
#define LOWPRI_TIER2_IO_PERIOD_SSD_MSECS 15
|
||
|
|
#define LOWPRI_TIER3_IO_PERIOD_SSD_MSECS 25
|
||
|
|
|
||
|
|
|
||
|
|
int throttle_windows_msecs[THROTTLE_LEVEL_END + 1] = {
|
||
|
|
0,
|
||
|
|
LOWPRI_TIER1_WINDOW_MSECS,
|
||
|
|
LOWPRI_TIER2_WINDOW_MSECS,
|
||
|
|
LOWPRI_TIER3_WINDOW_MSECS,
|
||
|
|
};
|
||
|
|
|
||
|
|
int throttle_io_period_msecs[THROTTLE_LEVEL_END + 1] = {
|
||
|
|
0,
|
||
|
|
LOWPRI_TIER1_IO_PERIOD_MSECS,
|
||
|
|
LOWPRI_TIER2_IO_PERIOD_MSECS,
|
||
|
|
LOWPRI_TIER3_IO_PERIOD_MSECS,
|
||
|
|
};
|
||
|
|
|
||
|
|
int throttle_io_period_ssd_msecs[THROTTLE_LEVEL_END + 1] = {
|
||
|
|
0,
|
||
|
|
LOWPRI_TIER1_IO_PERIOD_SSD_MSECS,
|
||
|
|
LOWPRI_TIER2_IO_PERIOD_SSD_MSECS,
|
||
|
|
LOWPRI_TIER3_IO_PERIOD_SSD_MSECS,
|
||
|
|
};
|
||
|
|
|
||
|
|
|
||
|
|
int throttled_count[THROTTLE_LEVEL_END + 1];
|
||
|
|
|
||
|
|
struct _throttle_io_info_t {
|
||
|
|
lck_mtx_t throttle_lock;
|
||
|
|
|
||
|
|
struct timeval throttle_last_write_timestamp;
|
||
|
|
struct timeval throttle_min_timer_deadline;
|
||
|
|
struct timeval throttle_window_start_timestamp[THROTTLE_LEVEL_END + 1]; /* window starts at both the beginning and completion of an I/O */
|
||
|
|
struct timeval throttle_last_IO_timestamp[THROTTLE_LEVEL_END + 1];
|
||
|
|
pid_t throttle_last_IO_pid[THROTTLE_LEVEL_END + 1];
|
||
|
|
struct timeval throttle_start_IO_period_timestamp[THROTTLE_LEVEL_END + 1];
|
||
|
|
int32_t throttle_inflight_count[THROTTLE_LEVEL_END + 1];
|
||
|
|
|
||
|
|
TAILQ_HEAD(, uthread) throttle_uthlist[THROTTLE_LEVEL_END + 1]; /* Lists of throttled uthreads */
|
||
|
|
int throttle_next_wake_level;
|
||
|
|
|
||
|
|
thread_call_t throttle_timer_call;
|
||
|
|
int32_t throttle_timer_ref;
|
||
|
|
int32_t throttle_timer_active;
|
||
|
|
|
||
|
|
int32_t throttle_io_count;
|
||
|
|
int32_t throttle_io_count_begin;
|
||
|
|
int *throttle_io_periods;
|
||
|
|
uint32_t throttle_io_period_num;
|
||
|
|
|
||
|
|
int32_t throttle_refcnt;
|
||
|
|
int32_t throttle_alloc;
|
||
|
|
int32_t throttle_disabled;
|
||
|
|
int32_t throttle_is_fusion_with_priority;
|
||
|
|
};
|
||
|
|
|
||
|
|
struct _throttle_io_info_t _throttle_io_info[LOWPRI_MAX_NUM_DEV];
|
||
|
|
|
||
|
|
|
||
|
|
int lowpri_throttle_enabled = 1;
|
||
|
|
|
||
|
|
|
||
|
|
static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level);
|
||
|
|
static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap);
|
||
|
|
static int throttle_get_thread_throttle_level(uthread_t ut);
|
||
|
|
static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier);
|
||
|
|
void throttle_info_mount_reset_period(mount_t mp, int isssd);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Trivial lookup routine that always fails.
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_lookup(struct vnop_lookup_args *ap)
|
||
|
|
{
|
||
|
|
*ap->a_vpp = NULL;
|
||
|
|
return ENOTDIR;
|
||
|
|
}
|
||
|
|
|
||
|
|
static void
|
||
|
|
set_blocksize(struct vnode *vp, dev_t dev)
|
||
|
|
{
|
||
|
|
int (*size)(dev_t);
|
||
|
|
int rsize;
|
||
|
|
|
||
|
|
if ((major(dev) < nblkdev) && (size = bdevsw[major(dev)].d_psize)) {
|
||
|
|
rsize = (*size)(dev);
|
||
|
|
if (rsize <= 0) { /* did size fail? */
|
||
|
|
vp->v_specsize = DEV_BSIZE;
|
||
|
|
} else {
|
||
|
|
vp->v_specsize = rsize;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
vp->v_specsize = DEV_BSIZE;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
void
|
||
|
|
set_fsblocksize(struct vnode *vp)
|
||
|
|
{
|
||
|
|
if (vp->v_type == VBLK) {
|
||
|
|
dev_t dev = (dev_t)vp->v_rdev;
|
||
|
|
int maj = major(dev);
|
||
|
|
|
||
|
|
if ((u_int)maj >= (u_int)nblkdev) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
vnode_lock(vp);
|
||
|
|
set_blocksize(vp, dev);
|
||
|
|
vnode_unlock(vp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void
|
||
|
|
spec_init_bsdunit(vnode_t vp, vfs_context_t ctx, const char* caller)
|
||
|
|
{
|
||
|
|
int isssd = 0;
|
||
|
|
uint64_t throttle_mask = 0;
|
||
|
|
uint32_t devbsdunit = 0;
|
||
|
|
|
||
|
|
if (VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ctx)) {
|
||
|
|
isssd = 0;
|
||
|
|
}
|
||
|
|
if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL)) {
|
||
|
|
throttle_mask = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (throttle_mask != 0) {
|
||
|
|
/*
|
||
|
|
* as a reasonable approximation, only use the lowest bit of the mask
|
||
|
|
* to generate a disk unit number
|
||
|
|
*/
|
||
|
|
devbsdunit = num_trailing_0(throttle_mask);
|
||
|
|
} else {
|
||
|
|
devbsdunit = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (vp->v_un.vu_specinfo->si_initted == 0) {
|
||
|
|
vnode_lock(vp);
|
||
|
|
if (vp->v_un.vu_specinfo->si_initted == 0) {
|
||
|
|
vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0;
|
||
|
|
vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
|
||
|
|
vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
|
||
|
|
vp->v_un.vu_specinfo->si_throttleable = 1;
|
||
|
|
vp->v_un.vu_specinfo->si_initted = 1;
|
||
|
|
}
|
||
|
|
vnode_unlock(vp);
|
||
|
|
printf("%s : si_devbsdunit initialized to (%d), throttle_mask is (0x%llx), isssd is (%d)\n",
|
||
|
|
caller, vp->v_un.vu_specinfo->si_devbsdunit,
|
||
|
|
vp->v_un.vu_specinfo->si_throttle_mask,
|
||
|
|
vp->v_un.vu_specinfo->si_isssd);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#define SPEC_INIT_BSDUNIT(vp, ctx) spec_init_bsdunit((vp), (ctx), __FUNCTION__)
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Open a special file.
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_open(struct vnop_open_args *ap)
|
||
|
|
{
|
||
|
|
struct proc *p = vfs_context_proc(ap->a_context);
|
||
|
|
kauth_cred_t cred = vfs_context_ucred(ap->a_context);
|
||
|
|
struct vnode *vp = ap->a_vp;
|
||
|
|
dev_t bdev, dev = (dev_t)vp->v_rdev;
|
||
|
|
int maj = major(dev);
|
||
|
|
int error;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Don't allow open if fs is mounted -nodev.
|
||
|
|
*/
|
||
|
|
if (vp->v_mount && (vp->v_mount->mnt_flag & MNT_NODEV)) {
|
||
|
|
return ENXIO;
|
||
|
|
}
|
||
|
|
|
||
|
|
switch (vp->v_type) {
|
||
|
|
case VCHR:
|
||
|
|
if ((u_int)maj >= (u_int)nchrdev) {
|
||
|
|
return ENXIO;
|
||
|
|
}
|
||
|
|
if (cred != FSCRED && (ap->a_mode & FWRITE)) {
|
||
|
|
#if 0
|
||
|
|
/*
|
||
|
|
* When running in very secure mode, do not allow
|
||
|
|
* opens for writing of any disk character devices.
|
||
|
|
*/
|
||
|
|
if (securelevel >= 2 && isdisk(dev, VCHR)) {
|
||
|
|
return EPERM;
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/* Never allow writing to /dev/mem or /dev/kmem */
|
||
|
|
if (iskmemdev(dev)) {
|
||
|
|
return EPERM;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* When running in secure mode, do not allow opens for
|
||
|
|
* writing of character devices whose corresponding block
|
||
|
|
* devices are currently mounted.
|
||
|
|
*/
|
||
|
|
if (securelevel >= 1) {
|
||
|
|
if ((bdev = chrtoblk(dev)) != NODEV && check_mountedon(bdev, VBLK, &error)) {
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
devsw_lock(dev, S_IFCHR);
|
||
|
|
error = (*cdevsw[maj].d_open)(dev, ap->a_mode, S_IFCHR, p);
|
||
|
|
|
||
|
|
if (error == 0) {
|
||
|
|
vp->v_specinfo->si_opencount++;
|
||
|
|
}
|
||
|
|
|
||
|
|
devsw_unlock(dev, S_IFCHR);
|
||
|
|
|
||
|
|
if (error == 0 && cdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
|
||
|
|
int isssd = 0;
|
||
|
|
uint64_t throttle_mask = 0;
|
||
|
|
uint32_t devbsdunit = 0;
|
||
|
|
|
||
|
|
if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL) == 0) {
|
||
|
|
if (throttle_mask != 0 &&
|
||
|
|
VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ap->a_context) == 0) {
|
||
|
|
/*
|
||
|
|
* as a reasonable approximation, only use the lowest bit of the mask
|
||
|
|
* to generate a disk unit number
|
||
|
|
*/
|
||
|
|
devbsdunit = num_trailing_0(throttle_mask);
|
||
|
|
|
||
|
|
vnode_lock(vp);
|
||
|
|
|
||
|
|
vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0;
|
||
|
|
vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
|
||
|
|
vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
|
||
|
|
vp->v_un.vu_specinfo->si_throttleable = 1;
|
||
|
|
vp->v_un.vu_specinfo->si_initted = 1;
|
||
|
|
|
||
|
|
vnode_unlock(vp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (vp->v_un.vu_specinfo->si_initted == 0) {
|
||
|
|
vnode_lock(vp);
|
||
|
|
vp->v_un.vu_specinfo->si_initted = 1;
|
||
|
|
vnode_unlock(vp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return error;
|
||
|
|
|
||
|
|
case VBLK:
|
||
|
|
if ((u_int)maj >= (u_int)nblkdev) {
|
||
|
|
return ENXIO;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* When running in very secure mode, do not allow
|
||
|
|
* opens for writing of any disk block devices.
|
||
|
|
*/
|
||
|
|
if (securelevel >= 2 && cred != FSCRED &&
|
||
|
|
(ap->a_mode & FWRITE) && bdevsw[maj].d_type == D_DISK) {
|
||
|
|
return EPERM;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* Do not allow opens of block devices that are
|
||
|
|
* currently mounted.
|
||
|
|
*/
|
||
|
|
if ((error = vfs_mountedon(vp))) {
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
|
||
|
|
devsw_lock(dev, S_IFBLK);
|
||
|
|
error = (*bdevsw[maj].d_open)(dev, ap->a_mode, S_IFBLK, p);
|
||
|
|
if (!error) {
|
||
|
|
vp->v_specinfo->si_opencount++;
|
||
|
|
}
|
||
|
|
devsw_unlock(dev, S_IFBLK);
|
||
|
|
|
||
|
|
if (!error) {
|
||
|
|
u_int64_t blkcnt;
|
||
|
|
u_int32_t blksize;
|
||
|
|
int setsize = 0;
|
||
|
|
u_int32_t size512 = 512;
|
||
|
|
|
||
|
|
if (bdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
|
||
|
|
SPEC_INIT_BSDUNIT(vp, ap->a_context);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!VNOP_IOCTL(vp, DKIOCGETBLOCKSIZE, (caddr_t)&blksize, 0, ap->a_context)) {
|
||
|
|
/* Switch to 512 byte sectors (temporarily) */
|
||
|
|
|
||
|
|
if (!VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&size512, FWRITE, ap->a_context)) {
|
||
|
|
/* Get the number of 512 byte physical blocks. */
|
||
|
|
if (!VNOP_IOCTL(vp, DKIOCGETBLOCKCOUNT, (caddr_t)&blkcnt, 0, ap->a_context)) {
|
||
|
|
setsize = 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/* If it doesn't set back, we can't recover */
|
||
|
|
if (VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&blksize, FWRITE, ap->a_context)) {
|
||
|
|
error = ENXIO;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
vnode_lock(vp);
|
||
|
|
set_blocksize(vp, dev);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Cache the size in bytes of the block device for later
|
||
|
|
* use by spec_write().
|
||
|
|
*/
|
||
|
|
if (setsize) {
|
||
|
|
vp->v_specdevsize = blkcnt * (u_int64_t)size512;
|
||
|
|
} else {
|
||
|
|
vp->v_specdevsize = (u_int64_t)0; /* Default: Can't get */
|
||
|
|
}
|
||
|
|
vnode_unlock(vp);
|
||
|
|
}
|
||
|
|
return error;
|
||
|
|
default:
|
||
|
|
panic("spec_open type");
|
||
|
|
}
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Vnode op for read
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_read(struct vnop_read_args *ap)
|
||
|
|
{
|
||
|
|
struct vnode *vp = ap->a_vp;
|
||
|
|
struct uio *uio = ap->a_uio;
|
||
|
|
struct buf *bp;
|
||
|
|
daddr64_t bn, nextbn;
|
||
|
|
long bscale;
|
||
|
|
int devBlockSize = 0;
|
||
|
|
size_t bsize, n, on;
|
||
|
|
int error = 0;
|
||
|
|
dev_t dev;
|
||
|
|
|
||
|
|
#if DIAGNOSTIC
|
||
|
|
if (uio->uio_rw != UIO_READ) {
|
||
|
|
panic("spec_read mode");
|
||
|
|
}
|
||
|
|
if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
|
||
|
|
panic("spec_read proc");
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
if (uio_resid(uio) == 0) {
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
switch (vp->v_type) {
|
||
|
|
case VCHR:
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *throttle_info = NULL;
|
||
|
|
int thread_throttle_level;
|
||
|
|
uint64_t blkno = 0;
|
||
|
|
uint32_t iolen = 0;
|
||
|
|
int ddisk = 0;
|
||
|
|
int ktrace_code = DKIO_READ;
|
||
|
|
devBlockSize = vp->v_specsize;
|
||
|
|
uintptr_t our_id = 0;
|
||
|
|
|
||
|
|
if (cdevsw[major(vp->v_rdev)].d_type == D_DISK) {
|
||
|
|
ddisk = 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
|
||
|
|
throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
|
||
|
|
thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (kdebug_enable && ddisk) {
|
||
|
|
if (devBlockSize == 0) {
|
||
|
|
devBlockSize = 512; // default sector size
|
||
|
|
}
|
||
|
|
|
||
|
|
if (uio_offset(uio) && devBlockSize) {
|
||
|
|
blkno = ((uint64_t) uio_offset(uio) / ((uint64_t)devBlockSize));
|
||
|
|
}
|
||
|
|
iolen = (int) uio_resid(uio);
|
||
|
|
our_id = (uintptr_t)thread_tid(current_thread());
|
||
|
|
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
|
||
|
|
(FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
|
||
|
|
vp->v_rdev, blkno, iolen, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
error = (*cdevsw[major(vp->v_rdev)].d_read)
|
||
|
|
(vp->v_rdev, uio, ap->a_ioflag);
|
||
|
|
|
||
|
|
|
||
|
|
if (kdebug_enable && ddisk) {
|
||
|
|
uint32_t residual = (uint32_t)uio_resid(uio);
|
||
|
|
ktrace_code |= DKIO_DONE;
|
||
|
|
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
|
||
|
|
(FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
|
||
|
|
(uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (throttle_info) {
|
||
|
|
throttle_info_end_io_internal(throttle_info, thread_throttle_level);
|
||
|
|
}
|
||
|
|
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
|
||
|
|
case VBLK:
|
||
|
|
if (uio->uio_offset < 0) {
|
||
|
|
return EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
dev = vp->v_rdev;
|
||
|
|
|
||
|
|
devBlockSize = vp->v_specsize;
|
||
|
|
|
||
|
|
if (devBlockSize > PAGE_SIZE) {
|
||
|
|
return EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
bscale = PAGE_SIZE / devBlockSize;
|
||
|
|
bsize = bscale * devBlockSize;
|
||
|
|
|
||
|
|
do {
|
||
|
|
on = uio->uio_offset % bsize;
|
||
|
|
|
||
|
|
bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~(bscale - 1));
|
||
|
|
|
||
|
|
if (vp->v_speclastr + bscale == bn) {
|
||
|
|
nextbn = bn + bscale;
|
||
|
|
error = buf_breadn(vp, bn, (int)bsize, &nextbn,
|
||
|
|
(int *)&bsize, 1, NOCRED, &bp);
|
||
|
|
} else {
|
||
|
|
error = buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
|
||
|
|
}
|
||
|
|
|
||
|
|
vnode_lock(vp);
|
||
|
|
vp->v_speclastr = bn;
|
||
|
|
vnode_unlock(vp);
|
||
|
|
|
||
|
|
n = bsize - buf_resid(bp);
|
||
|
|
if ((on > n) || error) {
|
||
|
|
if (!error) {
|
||
|
|
error = EINVAL;
|
||
|
|
}
|
||
|
|
buf_brelse(bp);
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
n = MIN((n - on), (size_t)uio_resid(uio));
|
||
|
|
|
||
|
|
error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
|
||
|
|
if (n + on == bsize) {
|
||
|
|
buf_markaged(bp);
|
||
|
|
}
|
||
|
|
buf_brelse(bp);
|
||
|
|
} while (error == 0 && uio_resid(uio) > 0 && n != 0);
|
||
|
|
return error;
|
||
|
|
|
||
|
|
default:
|
||
|
|
panic("spec_read type");
|
||
|
|
}
|
||
|
|
/* NOTREACHED */
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Vnode op for write
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_write(struct vnop_write_args *ap)
|
||
|
|
{
|
||
|
|
struct vnode *vp = ap->a_vp;
|
||
|
|
struct uio *uio = ap->a_uio;
|
||
|
|
struct buf *bp;
|
||
|
|
daddr64_t bn;
|
||
|
|
int blkmask, bscale;
|
||
|
|
int io_sync;
|
||
|
|
int devBlockSize = 0;
|
||
|
|
size_t bsize, n, on;
|
||
|
|
int error = 0;
|
||
|
|
dev_t dev;
|
||
|
|
|
||
|
|
#if DIAGNOSTIC
|
||
|
|
if (uio->uio_rw != UIO_WRITE) {
|
||
|
|
panic("spec_write mode");
|
||
|
|
}
|
||
|
|
if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
|
||
|
|
panic("spec_write proc");
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
switch (vp->v_type) {
|
||
|
|
case VCHR:
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *throttle_info = NULL;
|
||
|
|
int thread_throttle_level;
|
||
|
|
dev = vp->v_rdev;
|
||
|
|
devBlockSize = vp->v_specsize;
|
||
|
|
uint32_t iolen = 0;
|
||
|
|
uint64_t blkno = 0;
|
||
|
|
int ddisk = 0;
|
||
|
|
int ktrace_code = 0; // write is implied; read must be OR'd in.
|
||
|
|
uintptr_t our_id = 0;
|
||
|
|
|
||
|
|
if (cdevsw[major(dev)].d_type == D_DISK) {
|
||
|
|
ddisk = 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
|
||
|
|
throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
|
||
|
|
|
||
|
|
thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
|
||
|
|
|
||
|
|
microuptime(&throttle_info->throttle_last_write_timestamp);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (kdebug_enable && ddisk) {
|
||
|
|
if (devBlockSize == 0) {
|
||
|
|
devBlockSize = 512; // default sector size
|
||
|
|
}
|
||
|
|
if ((uio_offset(uio) != 0) && devBlockSize) {
|
||
|
|
blkno = ((uint64_t)uio_offset(uio)) / ((uint64_t)devBlockSize);
|
||
|
|
}
|
||
|
|
iolen = (int)uio_resid(uio);
|
||
|
|
our_id = (uintptr_t)thread_tid(current_thread());
|
||
|
|
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
|
||
|
|
(FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
|
||
|
|
vp->v_rdev, blkno, iolen, 0);
|
||
|
|
}
|
||
|
|
error = (*cdevsw[major(vp->v_rdev)].d_write)
|
||
|
|
(vp->v_rdev, uio, ap->a_ioflag);
|
||
|
|
|
||
|
|
if (kdebug_enable && ddisk) {
|
||
|
|
//emit the I/O completion
|
||
|
|
uint32_t residual = (uint32_t)uio_resid(uio);
|
||
|
|
ktrace_code |= DKIO_DONE;
|
||
|
|
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
|
||
|
|
(FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
|
||
|
|
(uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (throttle_info) {
|
||
|
|
throttle_info_end_io_internal(throttle_info, thread_throttle_level);
|
||
|
|
}
|
||
|
|
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
|
||
|
|
case VBLK:
|
||
|
|
if (uio_resid(uio) == 0) {
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
if (uio->uio_offset < 0) {
|
||
|
|
return EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
io_sync = (ap->a_ioflag & IO_SYNC);
|
||
|
|
|
||
|
|
dev = (vp->v_rdev);
|
||
|
|
|
||
|
|
devBlockSize = vp->v_specsize;
|
||
|
|
if (devBlockSize > PAGE_SIZE) {
|
||
|
|
return EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
bscale = PAGE_SIZE / devBlockSize;
|
||
|
|
blkmask = bscale - 1;
|
||
|
|
bsize = bscale * devBlockSize;
|
||
|
|
|
||
|
|
|
||
|
|
do {
|
||
|
|
bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~blkmask);
|
||
|
|
on = uio->uio_offset % bsize;
|
||
|
|
|
||
|
|
n = MIN((bsize - on), (size_t)uio_resid(uio));
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Use buf_getblk() as an optimization IFF:
|
||
|
|
*
|
||
|
|
* 1) We are reading exactly a block on a block
|
||
|
|
* aligned boundary
|
||
|
|
* 2) We know the size of the device from spec_open
|
||
|
|
* 3) The read doesn't span the end of the device
|
||
|
|
*
|
||
|
|
* Otherwise, we fall back on buf_bread().
|
||
|
|
*/
|
||
|
|
if (n == bsize &&
|
||
|
|
vp->v_specdevsize != (u_int64_t)0 &&
|
||
|
|
(uio->uio_offset + (u_int64_t)n) > vp->v_specdevsize) {
|
||
|
|
/* reduce the size of the read to what is there */
|
||
|
|
n = (uio->uio_offset + (u_int64_t)n) - vp->v_specdevsize;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (n == bsize) {
|
||
|
|
bp = buf_getblk(vp, bn, (int)bsize, 0, 0, BLK_WRITE);
|
||
|
|
} else {
|
||
|
|
error = (int)buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Translate downstream error for upstream, if needed */
|
||
|
|
if (!error) {
|
||
|
|
error = (int)buf_error(bp);
|
||
|
|
}
|
||
|
|
if (error) {
|
||
|
|
buf_brelse(bp);
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
n = MIN(n, bsize - buf_resid(bp));
|
||
|
|
|
||
|
|
error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
|
||
|
|
if (error) {
|
||
|
|
buf_brelse(bp);
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
buf_markaged(bp);
|
||
|
|
|
||
|
|
if (io_sync) {
|
||
|
|
error = buf_bwrite(bp);
|
||
|
|
} else {
|
||
|
|
if ((n + on) == bsize) {
|
||
|
|
error = buf_bawrite(bp);
|
||
|
|
} else {
|
||
|
|
error = buf_bdwrite(bp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} while (error == 0 && uio_resid(uio) > 0 && n != 0);
|
||
|
|
return error;
|
||
|
|
|
||
|
|
default:
|
||
|
|
panic("spec_write type");
|
||
|
|
}
|
||
|
|
/* NOTREACHED */
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Device ioctl operation.
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_ioctl(struct vnop_ioctl_args *ap)
|
||
|
|
{
|
||
|
|
proc_t p = vfs_context_proc(ap->a_context);
|
||
|
|
dev_t dev = ap->a_vp->v_rdev;
|
||
|
|
int retval = 0;
|
||
|
|
|
||
|
|
KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_START,
|
||
|
|
dev, ap->a_command, ap->a_fflag, ap->a_vp->v_type, 0);
|
||
|
|
|
||
|
|
switch (ap->a_vp->v_type) {
|
||
|
|
case VCHR:
|
||
|
|
retval = (*cdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data,
|
||
|
|
ap->a_fflag, p);
|
||
|
|
break;
|
||
|
|
|
||
|
|
case VBLK:
|
||
|
|
retval = (*bdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data, ap->a_fflag, p);
|
||
|
|
if (!retval && ap->a_command == DKIOCSETBLOCKSIZE) {
|
||
|
|
ap->a_vp->v_specsize = *(uint32_t *)ap->a_data;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
|
||
|
|
default:
|
||
|
|
panic("spec_ioctl");
|
||
|
|
/* NOTREACHED */
|
||
|
|
}
|
||
|
|
KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_END,
|
||
|
|
dev, ap->a_command, ap->a_fflag, retval, 0);
|
||
|
|
|
||
|
|
return retval;
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
spec_select(struct vnop_select_args *ap)
|
||
|
|
{
|
||
|
|
proc_t p = vfs_context_proc(ap->a_context);
|
||
|
|
dev_t dev;
|
||
|
|
|
||
|
|
switch (ap->a_vp->v_type) {
|
||
|
|
default:
|
||
|
|
return 1; /* XXX */
|
||
|
|
|
||
|
|
case VCHR:
|
||
|
|
dev = ap->a_vp->v_rdev;
|
||
|
|
return (*cdevsw[major(dev)].d_select)(dev, ap->a_which, ap->a_wql, p);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_qos_s *kev)
|
||
|
|
{
|
||
|
|
dev_t dev;
|
||
|
|
|
||
|
|
assert(vnode_ischr(vp));
|
||
|
|
|
||
|
|
dev = vnode_specrdev(vp);
|
||
|
|
|
||
|
|
#if NETWORKING
|
||
|
|
/*
|
||
|
|
* Try a bpf device, as defined in bsd/net/bpf.c
|
||
|
|
* If it doesn't error out the attach, then it
|
||
|
|
* claimed it. Otherwise, fall through and try
|
||
|
|
* other attaches.
|
||
|
|
*/
|
||
|
|
int32_t tmp_flags = kn->kn_flags;
|
||
|
|
int64_t tmp_sdata = kn->kn_sdata;
|
||
|
|
int res;
|
||
|
|
|
||
|
|
res = bpfkqfilter(dev, kn);
|
||
|
|
if ((kn->kn_flags & EV_ERROR) == 0) {
|
||
|
|
return res;
|
||
|
|
}
|
||
|
|
kn->kn_flags = tmp_flags;
|
||
|
|
kn->kn_sdata = tmp_sdata;
|
||
|
|
#endif
|
||
|
|
|
||
|
|
if (major(dev) >= nchrdev) {
|
||
|
|
knote_set_error(kn, ENXIO);
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE);
|
||
|
|
kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET);
|
||
|
|
|
||
|
|
if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) {
|
||
|
|
kn->kn_filtid = EVFILTID_PTSD;
|
||
|
|
return ptsd_kqfilter(dev, kn);
|
||
|
|
} else if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) {
|
||
|
|
kn->kn_filtid = EVFILTID_PTMX;
|
||
|
|
return ptmx_kqfilter(dev, kn);
|
||
|
|
} else if (cdevsw[major(dev)].d_type == D_TTY && kn->kn_vnode_kqok) {
|
||
|
|
/*
|
||
|
|
* TTYs from drivers that use struct ttys use their own filter
|
||
|
|
* routines. The PTC driver doesn't use the tty for character
|
||
|
|
* counts, so it must go through the select fallback.
|
||
|
|
*/
|
||
|
|
kn->kn_filtid = EVFILTID_TTY;
|
||
|
|
} else {
|
||
|
|
/* Try to attach to other char special devices */
|
||
|
|
kn->kn_filtid = EVFILTID_SPEC;
|
||
|
|
}
|
||
|
|
|
||
|
|
return knote_fops(kn)->f_attach(kn, kev);
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Synch buffers associated with a block device
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_fsync_internal(vnode_t vp, int waitfor, __unused vfs_context_t context)
|
||
|
|
{
|
||
|
|
if (vp->v_type == VCHR) {
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* Flush all dirty buffers associated with a block device.
|
||
|
|
*/
|
||
|
|
buf_flushdirtyblks(vp, (waitfor == MNT_WAIT || waitfor == MNT_DWAIT), 0, "spec_fsync");
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
spec_fsync(struct vnop_fsync_args *ap)
|
||
|
|
{
|
||
|
|
return spec_fsync_internal(ap->a_vp, ap->a_waitfor, ap->a_context);
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Just call the device strategy routine
|
||
|
|
*/
|
||
|
|
void throttle_init(void);
|
||
|
|
|
||
|
|
|
||
|
|
#if 0
|
||
|
|
#define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...) \
|
||
|
|
do { \
|
||
|
|
if ((debug_info)->alloc) \
|
||
|
|
printf("%s: "format, __FUNCTION__, ## args); \
|
||
|
|
} while(0)
|
||
|
|
|
||
|
|
#else
|
||
|
|
#define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...)
|
||
|
|
#endif
|
||
|
|
|
||
|
|
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER1], 0, "");
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER2], 0, "");
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER3], 0, "");
|
||
|
|
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER1], 0, "");
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER2], 0, "");
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER3], 0, "");
|
||
|
|
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER1], 0, "");
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER2], 0, "");
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER3], 0, "");
|
||
|
|
|
||
|
|
SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &lowpri_throttle_enabled, 0, "");
|
||
|
|
|
||
|
|
|
||
|
|
static LCK_GRP_DECLARE(throttle_lock_grp, "throttle I/O");
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* throttled I/O helper function
|
||
|
|
* convert the index of the lowest set bit to a device index
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
num_trailing_0(uint64_t n)
|
||
|
|
{
|
||
|
|
/*
|
||
|
|
* since in most cases the number of trailing 0s is very small,
|
||
|
|
* we simply counting sequentially from the lowest bit
|
||
|
|
*/
|
||
|
|
if (n == 0) {
|
||
|
|
return sizeof(n) * 8;
|
||
|
|
}
|
||
|
|
int count = 0;
|
||
|
|
while (!ISSET(n, 1)) {
|
||
|
|
n >>= 1;
|
||
|
|
++count;
|
||
|
|
}
|
||
|
|
return count;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Release the reference and if the item was allocated and this is the last
|
||
|
|
* reference then free it.
|
||
|
|
*
|
||
|
|
* This routine always returns the old value.
|
||
|
|
*/
|
||
|
|
static int
|
||
|
|
throttle_info_rel(struct _throttle_io_info_t *info)
|
||
|
|
{
|
||
|
|
SInt32 oldValue = OSDecrementAtomic(&info->throttle_refcnt);
|
||
|
|
|
||
|
|
DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
|
||
|
|
info, (int)(oldValue - 1), info );
|
||
|
|
|
||
|
|
/* The reference count just went negative, very bad */
|
||
|
|
if (oldValue == 0) {
|
||
|
|
panic("throttle info ref cnt went negative!");
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Once reference count is zero, no one else should be able to take a
|
||
|
|
* reference
|
||
|
|
*/
|
||
|
|
if ((oldValue == 1) && (info->throttle_alloc)) {
|
||
|
|
DEBUG_ALLOC_THROTTLE_INFO("Freeing info = %p\n", info);
|
||
|
|
|
||
|
|
lck_mtx_destroy(&info->throttle_lock, &throttle_lock_grp);
|
||
|
|
kfree_type(struct _throttle_io_info_t, info);
|
||
|
|
}
|
||
|
|
return oldValue;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Just take a reference on the throttle info structure.
|
||
|
|
*
|
||
|
|
* This routine always returns the old value.
|
||
|
|
*/
|
||
|
|
static SInt32
|
||
|
|
throttle_info_ref(struct _throttle_io_info_t *info)
|
||
|
|
{
|
||
|
|
SInt32 oldValue = OSIncrementAtomic(&info->throttle_refcnt);
|
||
|
|
|
||
|
|
DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
|
||
|
|
info, (int)(oldValue - 1), info );
|
||
|
|
/* Allocated items should never have a reference of zero */
|
||
|
|
if (info->throttle_alloc && (oldValue == 0)) {
|
||
|
|
panic("Taking a reference without calling create throttle info!");
|
||
|
|
}
|
||
|
|
|
||
|
|
return oldValue;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* on entry the throttle_lock is held...
|
||
|
|
* this function is responsible for taking
|
||
|
|
* and dropping the reference on the info
|
||
|
|
* structure which will keep it from going
|
||
|
|
* away while the timer is running if it
|
||
|
|
* happens to have been dynamically allocated by
|
||
|
|
* a network fileystem kext which is now trying
|
||
|
|
* to free it
|
||
|
|
*/
|
||
|
|
static uint32_t
|
||
|
|
throttle_timer_start(struct _throttle_io_info_t *info, boolean_t update_io_count, int wakelevel)
|
||
|
|
{
|
||
|
|
struct timeval elapsed;
|
||
|
|
struct timeval now;
|
||
|
|
struct timeval period;
|
||
|
|
uint64_t elapsed_msecs;
|
||
|
|
int throttle_level;
|
||
|
|
int level;
|
||
|
|
int msecs;
|
||
|
|
boolean_t throttled = FALSE;
|
||
|
|
boolean_t need_timer = FALSE;
|
||
|
|
|
||
|
|
microuptime(&now);
|
||
|
|
|
||
|
|
if (update_io_count == TRUE) {
|
||
|
|
info->throttle_io_count_begin = info->throttle_io_count;
|
||
|
|
info->throttle_io_period_num++;
|
||
|
|
|
||
|
|
while (wakelevel >= THROTTLE_LEVEL_THROTTLED) {
|
||
|
|
info->throttle_start_IO_period_timestamp[wakelevel--] = now;
|
||
|
|
}
|
||
|
|
|
||
|
|
info->throttle_min_timer_deadline = now;
|
||
|
|
|
||
|
|
msecs = info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED];
|
||
|
|
period.tv_sec = msecs / 1000;
|
||
|
|
period.tv_usec = (msecs % 1000) * 1000;
|
||
|
|
|
||
|
|
timevaladd(&info->throttle_min_timer_deadline, &period);
|
||
|
|
}
|
||
|
|
for (throttle_level = THROTTLE_LEVEL_START; throttle_level < THROTTLE_LEVEL_END; throttle_level++) {
|
||
|
|
elapsed = now;
|
||
|
|
timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
|
||
|
|
elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
|
||
|
|
|
||
|
|
for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
|
||
|
|
if (!TAILQ_EMPTY(&info->throttle_uthlist[level])) {
|
||
|
|
if (elapsed_msecs < (uint64_t)throttle_windows_msecs[level] || info->throttle_inflight_count[throttle_level]) {
|
||
|
|
/*
|
||
|
|
* we had an I/O occur at a higher priority tier within
|
||
|
|
* this tier's throttle window
|
||
|
|
*/
|
||
|
|
throttled = TRUE;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* we assume that the windows are the same or longer
|
||
|
|
* as we drop through the throttling tiers... thus
|
||
|
|
* we can stop looking once we run into a tier with
|
||
|
|
* threads to schedule regardless of whether it's
|
||
|
|
* still in its throttling window or not
|
||
|
|
*/
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (throttled == TRUE) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (throttled == TRUE) {
|
||
|
|
uint64_t deadline = 0;
|
||
|
|
struct timeval target;
|
||
|
|
struct timeval min_target;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* we've got at least one tier still in a throttled window
|
||
|
|
* so we need a timer running... compute the next deadline
|
||
|
|
* and schedule it
|
||
|
|
*/
|
||
|
|
for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
|
||
|
|
if (TAILQ_EMPTY(&info->throttle_uthlist[level])) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
target = info->throttle_start_IO_period_timestamp[level];
|
||
|
|
|
||
|
|
msecs = info->throttle_io_periods[level];
|
||
|
|
period.tv_sec = msecs / 1000;
|
||
|
|
period.tv_usec = (msecs % 1000) * 1000;
|
||
|
|
|
||
|
|
timevaladd(&target, &period);
|
||
|
|
|
||
|
|
if (need_timer == FALSE || timevalcmp(&target, &min_target, <)) {
|
||
|
|
min_target = target;
|
||
|
|
need_timer = TRUE;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (timevalcmp(&info->throttle_min_timer_deadline, &now, >)) {
|
||
|
|
if (timevalcmp(&info->throttle_min_timer_deadline, &min_target, >)) {
|
||
|
|
min_target = info->throttle_min_timer_deadline;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (info->throttle_timer_active) {
|
||
|
|
if (thread_call_cancel(info->throttle_timer_call) == FALSE) {
|
||
|
|
/*
|
||
|
|
* couldn't kill the timer because it's already
|
||
|
|
* been dispatched, so don't try to start a new
|
||
|
|
* one... once we drop the lock, the timer will
|
||
|
|
* proceed and eventually re-run this function
|
||
|
|
*/
|
||
|
|
need_timer = FALSE;
|
||
|
|
} else {
|
||
|
|
info->throttle_timer_active = 0;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (need_timer == TRUE) {
|
||
|
|
/*
|
||
|
|
* This is defined as an int (32-bit) rather than a 64-bit
|
||
|
|
* value because it would need a really big period in the
|
||
|
|
* order of ~500 days to overflow this. So, we let this be
|
||
|
|
* 32-bit which allows us to use the clock_interval_to_deadline()
|
||
|
|
* routine.
|
||
|
|
*/
|
||
|
|
int target_msecs;
|
||
|
|
|
||
|
|
if (info->throttle_timer_ref == 0) {
|
||
|
|
/*
|
||
|
|
* take a reference for the timer
|
||
|
|
*/
|
||
|
|
throttle_info_ref(info);
|
||
|
|
|
||
|
|
info->throttle_timer_ref = 1;
|
||
|
|
}
|
||
|
|
elapsed = min_target;
|
||
|
|
timevalsub(&elapsed, &now);
|
||
|
|
target_msecs = (int)(elapsed.tv_sec * 1000 + elapsed.tv_usec / 1000);
|
||
|
|
|
||
|
|
if (target_msecs <= 0) {
|
||
|
|
/*
|
||
|
|
* we may have computed a deadline slightly in the past
|
||
|
|
* due to various factors... if so, just set the timer
|
||
|
|
* to go off in the near future (we don't need to be precise)
|
||
|
|
*/
|
||
|
|
target_msecs = 1;
|
||
|
|
}
|
||
|
|
clock_interval_to_deadline(target_msecs, 1000000, &deadline);
|
||
|
|
|
||
|
|
thread_call_enter_delayed(info->throttle_timer_call, deadline);
|
||
|
|
info->throttle_timer_active = 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return throttle_level;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
static void
|
||
|
|
throttle_timer(struct _throttle_io_info_t *info, __unused thread_call_param_t p)
|
||
|
|
{
|
||
|
|
uthread_t ut, utlist;
|
||
|
|
struct timeval elapsed;
|
||
|
|
struct timeval now;
|
||
|
|
uint64_t elapsed_msecs;
|
||
|
|
int throttle_level;
|
||
|
|
int level;
|
||
|
|
int wake_level;
|
||
|
|
caddr_t wake_address = NULL;
|
||
|
|
boolean_t update_io_count = FALSE;
|
||
|
|
boolean_t need_wakeup = FALSE;
|
||
|
|
boolean_t need_release = FALSE;
|
||
|
|
|
||
|
|
ut = NULL;
|
||
|
|
lck_mtx_lock(&info->throttle_lock);
|
||
|
|
|
||
|
|
info->throttle_timer_active = 0;
|
||
|
|
microuptime(&now);
|
||
|
|
|
||
|
|
elapsed = now;
|
||
|
|
timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[THROTTLE_LEVEL_THROTTLED]);
|
||
|
|
elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
|
||
|
|
|
||
|
|
if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED]) {
|
||
|
|
wake_level = info->throttle_next_wake_level;
|
||
|
|
|
||
|
|
for (level = THROTTLE_LEVEL_START; level < THROTTLE_LEVEL_END; level++) {
|
||
|
|
elapsed = now;
|
||
|
|
timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[wake_level]);
|
||
|
|
elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
|
||
|
|
|
||
|
|
if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[wake_level] && !TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
|
||
|
|
/*
|
||
|
|
* we're closing out the current IO period...
|
||
|
|
* if we have a waiting thread, wake it up
|
||
|
|
* after we have reset the I/O window info
|
||
|
|
*/
|
||
|
|
need_wakeup = TRUE;
|
||
|
|
update_io_count = TRUE;
|
||
|
|
|
||
|
|
info->throttle_next_wake_level = wake_level - 1;
|
||
|
|
|
||
|
|
if (info->throttle_next_wake_level == THROTTLE_LEVEL_START) {
|
||
|
|
info->throttle_next_wake_level = THROTTLE_LEVEL_END;
|
||
|
|
}
|
||
|
|
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
wake_level--;
|
||
|
|
|
||
|
|
if (wake_level == THROTTLE_LEVEL_START) {
|
||
|
|
wake_level = THROTTLE_LEVEL_END;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (need_wakeup == TRUE) {
|
||
|
|
if (!TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
|
||
|
|
ut = (uthread_t)TAILQ_FIRST(&info->throttle_uthlist[wake_level]);
|
||
|
|
TAILQ_REMOVE(&info->throttle_uthlist[wake_level], ut, uu_throttlelist);
|
||
|
|
ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
|
||
|
|
ut->uu_is_throttled = false;
|
||
|
|
|
||
|
|
wake_address = (caddr_t)&ut->uu_on_throttlelist;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
wake_level = THROTTLE_LEVEL_START;
|
||
|
|
}
|
||
|
|
|
||
|
|
throttle_level = throttle_timer_start(info, update_io_count, wake_level);
|
||
|
|
|
||
|
|
if (wake_address != NULL) {
|
||
|
|
wakeup(wake_address);
|
||
|
|
}
|
||
|
|
|
||
|
|
for (level = THROTTLE_LEVEL_THROTTLED; level <= throttle_level; level++) {
|
||
|
|
TAILQ_FOREACH_SAFE(ut, &info->throttle_uthlist[level], uu_throttlelist, utlist) {
|
||
|
|
TAILQ_REMOVE(&info->throttle_uthlist[level], ut, uu_throttlelist);
|
||
|
|
ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
|
||
|
|
ut->uu_is_throttled = false;
|
||
|
|
|
||
|
|
wakeup(&ut->uu_on_throttlelist);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (info->throttle_timer_active == 0 && info->throttle_timer_ref) {
|
||
|
|
info->throttle_timer_ref = 0;
|
||
|
|
need_release = TRUE;
|
||
|
|
}
|
||
|
|
lck_mtx_unlock(&info->throttle_lock);
|
||
|
|
|
||
|
|
if (need_release == TRUE) {
|
||
|
|
throttle_info_rel(info);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
static int
|
||
|
|
throttle_add_to_list(struct _throttle_io_info_t *info, uthread_t ut, int mylevel, boolean_t insert_tail)
|
||
|
|
{
|
||
|
|
boolean_t start_timer = FALSE;
|
||
|
|
int level = THROTTLE_LEVEL_START;
|
||
|
|
|
||
|
|
if (TAILQ_EMPTY(&info->throttle_uthlist[mylevel])) {
|
||
|
|
info->throttle_start_IO_period_timestamp[mylevel] = info->throttle_last_IO_timestamp[mylevel];
|
||
|
|
start_timer = TRUE;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (insert_tail == TRUE) {
|
||
|
|
TAILQ_INSERT_TAIL(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
|
||
|
|
} else {
|
||
|
|
TAILQ_INSERT_HEAD(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
|
||
|
|
}
|
||
|
|
|
||
|
|
ut->uu_on_throttlelist = (int8_t)mylevel;
|
||
|
|
|
||
|
|
if (start_timer == TRUE) {
|
||
|
|
/* we may need to start or rearm the timer */
|
||
|
|
level = throttle_timer_start(info, FALSE, THROTTLE_LEVEL_START);
|
||
|
|
|
||
|
|
if (level == THROTTLE_LEVEL_END) {
|
||
|
|
if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
|
||
|
|
TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
|
||
|
|
|
||
|
|
ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return level;
|
||
|
|
}
|
||
|
|
|
||
|
|
static void
|
||
|
|
throttle_init_throttle_window(void)
|
||
|
|
{
|
||
|
|
int throttle_window_size;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* The hierarchy of throttle window values is as follows:
|
||
|
|
* - Global defaults
|
||
|
|
* - Device tree properties
|
||
|
|
* - Boot-args
|
||
|
|
* All values are specified in msecs.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#if (XNU_TARGET_OS_OSX && __arm64__)
|
||
|
|
/*
|
||
|
|
* IO Tier EDT overrides are meant for
|
||
|
|
* some arm platforms but not for
|
||
|
|
* macs.
|
||
|
|
*/
|
||
|
|
#else /* (XNU_TARGET_OS_OSX && __arm64__) */
|
||
|
|
/* Override global values with device-tree properties */
|
||
|
|
if (PE_get_default("kern.io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
|
||
|
|
throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_get_default("kern.io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
|
||
|
|
throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_get_default("kern.io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
|
||
|
|
throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
|
||
|
|
}
|
||
|
|
#endif /* (XNU_TARGET_OS_OSX && __arm64__) */
|
||
|
|
|
||
|
|
/* Override with boot-args */
|
||
|
|
if (PE_parse_boot_argn("io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
|
||
|
|
throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_parse_boot_argn("io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
|
||
|
|
throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_parse_boot_argn("io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
|
||
|
|
throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void
|
||
|
|
throttle_init_throttle_period(struct _throttle_io_info_t *info, boolean_t isssd)
|
||
|
|
{
|
||
|
|
int throttle_period_size;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* The hierarchy of throttle period values is as follows:
|
||
|
|
* - Global defaults
|
||
|
|
* - Device tree properties
|
||
|
|
* - Boot-args
|
||
|
|
* All values are specified in msecs.
|
||
|
|
*/
|
||
|
|
|
||
|
|
/* Assign global defaults */
|
||
|
|
if ((isssd == TRUE) && (info->throttle_is_fusion_with_priority == 0)) {
|
||
|
|
info->throttle_io_periods = &throttle_io_period_ssd_msecs[0];
|
||
|
|
} else {
|
||
|
|
info->throttle_io_periods = &throttle_io_period_msecs[0];
|
||
|
|
}
|
||
|
|
|
||
|
|
#if (XNU_TARGET_OS_OSX && __arm64__)
|
||
|
|
/*
|
||
|
|
* IO Tier EDT overrides are meant for
|
||
|
|
* some arm platforms but not for
|
||
|
|
* macs.
|
||
|
|
*/
|
||
|
|
#else /* (XNU_TARGET_OS_OSX && __arm64__) */
|
||
|
|
/* Override global values with device-tree properties */
|
||
|
|
if (PE_get_default("kern.io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
|
||
|
|
info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_get_default("kern.io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
|
||
|
|
info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_get_default("kern.io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
|
||
|
|
info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
|
||
|
|
}
|
||
|
|
#endif /* (XNU_TARGET_OS_OSX && __arm64__) */
|
||
|
|
|
||
|
|
/* Override with boot-args */
|
||
|
|
if (PE_parse_boot_argn("io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
|
||
|
|
info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_parse_boot_argn("io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
|
||
|
|
info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (PE_parse_boot_argn("io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
|
||
|
|
info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#if CONFIG_IOSCHED
|
||
|
|
extern void vm_io_reprioritize_init(void);
|
||
|
|
int iosched_enabled = 1;
|
||
|
|
#endif
|
||
|
|
|
||
|
|
void
|
||
|
|
throttle_init(void)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
int i;
|
||
|
|
int level;
|
||
|
|
#if CONFIG_IOSCHED
|
||
|
|
int iosched;
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/* Update throttle parameters based on device tree configuration */
|
||
|
|
throttle_init_throttle_window();
|
||
|
|
|
||
|
|
for (i = 0; i < LOWPRI_MAX_NUM_DEV; i++) {
|
||
|
|
info = &_throttle_io_info[i];
|
||
|
|
|
||
|
|
lck_mtx_init(&info->throttle_lock, &throttle_lock_grp, LCK_ATTR_NULL);
|
||
|
|
info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
|
||
|
|
|
||
|
|
for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
|
||
|
|
TAILQ_INIT(&info->throttle_uthlist[level]);
|
||
|
|
info->throttle_last_IO_pid[level] = 0;
|
||
|
|
info->throttle_inflight_count[level] = 0;
|
||
|
|
}
|
||
|
|
info->throttle_next_wake_level = THROTTLE_LEVEL_END;
|
||
|
|
info->throttle_disabled = 0;
|
||
|
|
info->throttle_is_fusion_with_priority = 0;
|
||
|
|
}
|
||
|
|
#if CONFIG_IOSCHED
|
||
|
|
if (PE_parse_boot_argn("iosched", &iosched, sizeof(iosched))) {
|
||
|
|
iosched_enabled = iosched;
|
||
|
|
}
|
||
|
|
if (iosched_enabled) {
|
||
|
|
/* Initialize I/O Reprioritization mechanism */
|
||
|
|
vm_io_reprioritize_init();
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
|
||
|
|
void
|
||
|
|
sys_override_io_throttle(boolean_t enable_override)
|
||
|
|
{
|
||
|
|
if (enable_override) {
|
||
|
|
lowpri_throttle_enabled = 0;
|
||
|
|
} else {
|
||
|
|
lowpri_throttle_enabled = 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int rethrottle_wakeups = 0;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* the uu_rethrottle_lock is used to synchronize this function
|
||
|
|
* with "throttle_lowpri_io" which is where a throttled thread
|
||
|
|
* will block... that function will grab this lock before beginning
|
||
|
|
* it's decision making process concerning the need to block, and
|
||
|
|
* hold it through the assert_wait. When that thread is awakened
|
||
|
|
* for any reason (timer or rethrottle), it will reacquire the
|
||
|
|
* uu_rethrottle_lock before determining if it really is ok for
|
||
|
|
* it to now run. This is the point at which the thread could
|
||
|
|
* enter a different throttling queue and reblock or return from
|
||
|
|
* the throttle w/o having waited out it's entire throttle if
|
||
|
|
* the rethrottle has now moved it out of any currently
|
||
|
|
* active throttle window.
|
||
|
|
*
|
||
|
|
*
|
||
|
|
* NOTES:
|
||
|
|
* 1 - This may be called with the task lock held.
|
||
|
|
* 2 - This may be called with preemption and interrupts disabled
|
||
|
|
* in the kqueue wakeup path so we can't take the throttle_lock which is a mutex
|
||
|
|
* 3 - This cannot safely dereference uu_throttle_info, as it may
|
||
|
|
* get deallocated out from under us
|
||
|
|
*/
|
||
|
|
|
||
|
|
void
|
||
|
|
rethrottle_thread(uthread_t ut)
|
||
|
|
{
|
||
|
|
/*
|
||
|
|
* If uthread doesn't have throttle state, then there's no chance
|
||
|
|
* of it needing a rethrottle.
|
||
|
|
*/
|
||
|
|
if (ut->uu_throttle_info == NULL) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
boolean_t s = ml_set_interrupts_enabled(FALSE);
|
||
|
|
lck_spin_lock(&ut->uu_rethrottle_lock);
|
||
|
|
|
||
|
|
if (!ut->uu_is_throttled) {
|
||
|
|
ut->uu_was_rethrottled = true;
|
||
|
|
} else {
|
||
|
|
int my_new_level = throttle_get_thread_throttle_level(ut);
|
||
|
|
|
||
|
|
if (my_new_level != ut->uu_on_throttlelist) {
|
||
|
|
/*
|
||
|
|
* ut is currently blocked (as indicated by
|
||
|
|
* ut->uu_is_throttled == true)
|
||
|
|
* and we're changing it's throttle level, so
|
||
|
|
* we need to wake it up.
|
||
|
|
*/
|
||
|
|
ut->uu_is_throttled = false;
|
||
|
|
wakeup(&ut->uu_on_throttlelist);
|
||
|
|
|
||
|
|
rethrottle_wakeups++;
|
||
|
|
KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 102)),
|
||
|
|
uthread_tid(ut), ut->uu_on_throttlelist, my_new_level, 0, 0);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
lck_spin_unlock(&ut->uu_rethrottle_lock);
|
||
|
|
ml_set_interrupts_enabled(s);
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* Create and take a reference on a throttle info structure and return a
|
||
|
|
* pointer for the file system to use when calling throttle_info_update.
|
||
|
|
* Calling file system must have a matching release for every create.
|
||
|
|
*/
|
||
|
|
void *
|
||
|
|
throttle_info_create(void)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
int level;
|
||
|
|
|
||
|
|
info = kalloc_type(struct _throttle_io_info_t,
|
||
|
|
Z_ZERO | Z_WAITOK | Z_NOFAIL);
|
||
|
|
/* Mark that this one was allocated and needs to be freed */
|
||
|
|
DEBUG_ALLOC_THROTTLE_INFO("Creating info = %p\n", info, info );
|
||
|
|
info->throttle_alloc = TRUE;
|
||
|
|
|
||
|
|
lck_mtx_init(&info->throttle_lock, &throttle_lock_grp, LCK_ATTR_NULL);
|
||
|
|
info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
|
||
|
|
|
||
|
|
for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
|
||
|
|
TAILQ_INIT(&info->throttle_uthlist[level]);
|
||
|
|
}
|
||
|
|
info->throttle_next_wake_level = THROTTLE_LEVEL_END;
|
||
|
|
|
||
|
|
/* Take a reference */
|
||
|
|
OSIncrementAtomic(&info->throttle_refcnt);
|
||
|
|
return info;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* Release the throttle info pointer if all the reference are gone. Should be
|
||
|
|
* called to release reference taken by throttle_info_create
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_release(void *throttle_info)
|
||
|
|
{
|
||
|
|
DEBUG_ALLOC_THROTTLE_INFO("Releaseing info = %p\n",
|
||
|
|
(struct _throttle_io_info_t *)throttle_info,
|
||
|
|
(struct _throttle_io_info_t *)throttle_info);
|
||
|
|
if (throttle_info) { /* Just to be careful */
|
||
|
|
throttle_info_rel(throttle_info);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* File Systems that create an info structure, need to call this routine in
|
||
|
|
* their mount routine (used by cluster code). File Systems that call this in
|
||
|
|
* their mount routines must call throttle_info_mount_rel in their unmount
|
||
|
|
* routines.
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_mount_ref(mount_t mp, void *throttle_info)
|
||
|
|
{
|
||
|
|
if ((throttle_info == NULL) || (mp == NULL)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
throttle_info_ref(throttle_info);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* We already have a reference release it before adding the new one
|
||
|
|
*/
|
||
|
|
if (mp->mnt_throttle_info) {
|
||
|
|
throttle_info_rel(mp->mnt_throttle_info);
|
||
|
|
}
|
||
|
|
mp->mnt_throttle_info = throttle_info;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Private KPI routine
|
||
|
|
*
|
||
|
|
* return a handle for accessing throttle_info given a throttle_mask. The
|
||
|
|
* handle must be released by throttle_info_rel_by_mask
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
throttle_info_ref_by_mask(uint64_t throttle_mask, throttle_info_handle_t *throttle_info_handle)
|
||
|
|
{
|
||
|
|
int dev_index;
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* The 'throttle_mask' is not expected to be 0 otherwise num_trailing_0()
|
||
|
|
* would return value of 64 and this will cause '_throttle_io_info' to
|
||
|
|
* go out of bounds as '_throttle_io_info' is only LOWPRI_MAX_NUM_DEV (64)
|
||
|
|
* elements long.
|
||
|
|
*/
|
||
|
|
if (throttle_info_handle == NULL || throttle_mask == 0) {
|
||
|
|
return EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
dev_index = num_trailing_0(throttle_mask);
|
||
|
|
info = &_throttle_io_info[dev_index];
|
||
|
|
throttle_info_ref(info);
|
||
|
|
*(struct _throttle_io_info_t**)throttle_info_handle = info;
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Private KPI routine
|
||
|
|
*
|
||
|
|
* release the handle obtained by throttle_info_ref_by_mask
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle)
|
||
|
|
{
|
||
|
|
/*
|
||
|
|
* for now the handle is just a pointer to _throttle_io_info_t
|
||
|
|
*/
|
||
|
|
throttle_info_rel((struct _throttle_io_info_t*)throttle_info_handle);
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* File Systems that throttle_info_mount_ref, must call this routine in their
|
||
|
|
* umount routine.
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_mount_rel(mount_t mp)
|
||
|
|
{
|
||
|
|
if (mp->mnt_throttle_info) {
|
||
|
|
throttle_info_rel(mp->mnt_throttle_info);
|
||
|
|
}
|
||
|
|
mp->mnt_throttle_info = NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Reset throttling periods for the given mount point
|
||
|
|
*
|
||
|
|
* private interface used by disk conditioner to reset
|
||
|
|
* throttling periods when 'is_ssd' status changes
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_mount_reset_period(mount_t mp, int isssd)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
|
||
|
|
if (mp == NULL) {
|
||
|
|
info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
|
||
|
|
} else if (mp->mnt_throttle_info == NULL) {
|
||
|
|
info = &_throttle_io_info[mp->mnt_devbsdunit];
|
||
|
|
} else {
|
||
|
|
info = mp->mnt_throttle_info;
|
||
|
|
}
|
||
|
|
|
||
|
|
throttle_init_throttle_period(info, isssd);
|
||
|
|
}
|
||
|
|
|
||
|
|
void
|
||
|
|
throttle_info_get_last_io_time(mount_t mp, struct timeval *tv)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
|
||
|
|
if (mp == NULL) {
|
||
|
|
info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
|
||
|
|
} else if (mp->mnt_throttle_info == NULL) {
|
||
|
|
info = &_throttle_io_info[mp->mnt_devbsdunit];
|
||
|
|
} else {
|
||
|
|
info = mp->mnt_throttle_info;
|
||
|
|
}
|
||
|
|
|
||
|
|
*tv = info->throttle_last_write_timestamp;
|
||
|
|
}
|
||
|
|
|
||
|
|
void
|
||
|
|
update_last_io_time(mount_t mp)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
|
||
|
|
if (mp == NULL) {
|
||
|
|
info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
|
||
|
|
} else if (mp->mnt_throttle_info == NULL) {
|
||
|
|
info = &_throttle_io_info[mp->mnt_devbsdunit];
|
||
|
|
} else {
|
||
|
|
info = mp->mnt_throttle_info;
|
||
|
|
}
|
||
|
|
|
||
|
|
microuptime(&info->throttle_last_write_timestamp);
|
||
|
|
if (mp != NULL) {
|
||
|
|
mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
throttle_get_io_policy(uthread_t *ut)
|
||
|
|
{
|
||
|
|
if (ut != NULL) {
|
||
|
|
*ut = current_uthread();
|
||
|
|
}
|
||
|
|
|
||
|
|
return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
throttle_get_passive_io_policy(uthread_t *ut)
|
||
|
|
{
|
||
|
|
if (ut != NULL) {
|
||
|
|
*ut = current_uthread();
|
||
|
|
}
|
||
|
|
|
||
|
|
return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_PASSIVE_IO);
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
static int
|
||
|
|
throttle_get_thread_throttle_level(uthread_t ut)
|
||
|
|
{
|
||
|
|
uthread_t *ut_p = (ut == NULL) ? &ut : NULL;
|
||
|
|
int io_tier = throttle_get_io_policy(ut_p);
|
||
|
|
|
||
|
|
return throttle_get_thread_throttle_level_internal(ut, io_tier);
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Return a throttle level given an existing I/O tier (such as returned by throttle_get_io_policy)
|
||
|
|
*/
|
||
|
|
static int
|
||
|
|
throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier)
|
||
|
|
{
|
||
|
|
int thread_throttle_level = io_tier;
|
||
|
|
int user_idle_level;
|
||
|
|
|
||
|
|
assert(ut != NULL);
|
||
|
|
|
||
|
|
/* Bootcache misses should always be throttled */
|
||
|
|
if (ut->uu_throttle_bc) {
|
||
|
|
thread_throttle_level = THROTTLE_LEVEL_TIER3;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Issue tier3 I/O as tier2 when the user is idle
|
||
|
|
* to allow maintenance tasks to make more progress.
|
||
|
|
*
|
||
|
|
* Assume any positive idle level is enough... for now it's
|
||
|
|
* only ever 0 or 128 but this is not defined anywhere.
|
||
|
|
*/
|
||
|
|
if (thread_throttle_level >= THROTTLE_LEVEL_TIER3) {
|
||
|
|
user_idle_level = timer_get_user_idle_level();
|
||
|
|
if (user_idle_level > 0) {
|
||
|
|
thread_throttle_level--;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return thread_throttle_level;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* I/O will be throttled if either of the following are true:
|
||
|
|
* - Higher tiers have in-flight I/O
|
||
|
|
* - The time delta since the last start/completion of a higher tier is within the throttle window interval
|
||
|
|
*
|
||
|
|
* In-flight I/O is bookended by throttle_info_update_internal/throttle_info_end_io_internal
|
||
|
|
*/
|
||
|
|
static int
|
||
|
|
throttle_io_will_be_throttled_internal(void * throttle_info, int * mylevel, int * throttling_level)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info = throttle_info;
|
||
|
|
struct timeval elapsed;
|
||
|
|
struct timeval now;
|
||
|
|
uint64_t elapsed_msecs;
|
||
|
|
int thread_throttle_level;
|
||
|
|
int throttle_level;
|
||
|
|
|
||
|
|
if ((thread_throttle_level = throttle_get_thread_throttle_level(NULL)) < THROTTLE_LEVEL_THROTTLED) {
|
||
|
|
return THROTTLE_DISENGAGED;
|
||
|
|
}
|
||
|
|
|
||
|
|
microuptime(&now);
|
||
|
|
|
||
|
|
for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
|
||
|
|
if (info->throttle_inflight_count[throttle_level]) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
elapsed = now;
|
||
|
|
timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
|
||
|
|
elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
|
||
|
|
|
||
|
|
if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (throttle_level >= thread_throttle_level) {
|
||
|
|
/*
|
||
|
|
* we're beyond all of the throttle windows
|
||
|
|
* that affect the throttle level of this thread,
|
||
|
|
* so go ahead and treat as normal I/O
|
||
|
|
*/
|
||
|
|
return THROTTLE_DISENGAGED;
|
||
|
|
}
|
||
|
|
if (mylevel) {
|
||
|
|
*mylevel = thread_throttle_level;
|
||
|
|
}
|
||
|
|
if (throttling_level) {
|
||
|
|
*throttling_level = throttle_level;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (info->throttle_io_count != info->throttle_io_count_begin) {
|
||
|
|
/*
|
||
|
|
* we've already issued at least one throttleable I/O
|
||
|
|
* in the current I/O window, so avoid issuing another one
|
||
|
|
*/
|
||
|
|
return THROTTLE_NOW;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* we're in the throttle window, so
|
||
|
|
* cut the I/O size back
|
||
|
|
*/
|
||
|
|
return THROTTLE_ENGAGED;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* If we have a mount point and it has a throttle info pointer then
|
||
|
|
* use it to do the check, otherwise use the device unit number to find
|
||
|
|
* the correct throttle info array element.
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
throttle_io_will_be_throttled(__unused int lowpri_window_msecs, mount_t mp)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Should we just return zero if no mount point
|
||
|
|
*/
|
||
|
|
if (mp == NULL) {
|
||
|
|
info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
|
||
|
|
} else if (mp->mnt_throttle_info == NULL) {
|
||
|
|
info = &_throttle_io_info[mp->mnt_devbsdunit];
|
||
|
|
} else {
|
||
|
|
info = mp->mnt_throttle_info;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (info->throttle_is_fusion_with_priority) {
|
||
|
|
uthread_t ut = current_uthread();
|
||
|
|
if (ut->uu_lowpri_window == 0) {
|
||
|
|
return THROTTLE_DISENGAGED;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (info->throttle_disabled) {
|
||
|
|
return THROTTLE_DISENGAGED;
|
||
|
|
} else {
|
||
|
|
return throttle_io_will_be_throttled_internal(info, NULL, NULL);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Routine to increment I/O throttling counters maintained in the proc
|
||
|
|
*/
|
||
|
|
|
||
|
|
static void
|
||
|
|
throttle_update_proc_stats(pid_t throttling_pid, int count)
|
||
|
|
{
|
||
|
|
proc_t throttling_proc;
|
||
|
|
proc_t throttled_proc = current_proc();
|
||
|
|
|
||
|
|
/* The throttled_proc is always the current proc; so we are not concerned with refs */
|
||
|
|
OSAddAtomic64(count, &(throttled_proc->was_throttled));
|
||
|
|
|
||
|
|
/* The throttling pid might have exited by now */
|
||
|
|
throttling_proc = proc_find(throttling_pid);
|
||
|
|
if (throttling_proc != PROC_NULL) {
|
||
|
|
OSAddAtomic64(count, &(throttling_proc->did_throttle));
|
||
|
|
proc_rele(throttling_proc);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Block until woken up by the throttle timer or by a rethrottle call.
|
||
|
|
* As long as we hold the throttle_lock while querying the throttle tier, we're
|
||
|
|
* safe against seeing an old throttle tier after a rethrottle.
|
||
|
|
*/
|
||
|
|
uint32_t
|
||
|
|
throttle_lowpri_io(int sleep_amount)
|
||
|
|
{
|
||
|
|
uthread_t ut;
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
int throttle_type = 0;
|
||
|
|
int mylevel = 0;
|
||
|
|
int throttling_level = THROTTLE_LEVEL_NONE;
|
||
|
|
int sleep_cnt = 0;
|
||
|
|
uint32_t throttle_io_period_num = 0;
|
||
|
|
boolean_t insert_tail = TRUE;
|
||
|
|
boolean_t s;
|
||
|
|
|
||
|
|
ut = current_uthread();
|
||
|
|
|
||
|
|
if (ut->uu_lowpri_window == 0) {
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
info = ut->uu_throttle_info;
|
||
|
|
|
||
|
|
if (info == NULL) {
|
||
|
|
ut->uu_throttle_bc = false;
|
||
|
|
ut->uu_lowpri_window = 0;
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
lck_mtx_lock(&info->throttle_lock);
|
||
|
|
assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
|
||
|
|
|
||
|
|
if (sleep_amount == 0) {
|
||
|
|
goto done;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (sleep_amount == 1 && !ut->uu_throttle_bc) {
|
||
|
|
sleep_amount = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
throttle_io_period_num = info->throttle_io_period_num;
|
||
|
|
|
||
|
|
ut->uu_was_rethrottled = false;
|
||
|
|
|
||
|
|
while ((throttle_type = throttle_io_will_be_throttled_internal(info, &mylevel, &throttling_level))) {
|
||
|
|
if (throttle_type == THROTTLE_ENGAGED) {
|
||
|
|
if (sleep_amount == 0) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
if (info->throttle_io_period_num < throttle_io_period_num) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
if ((info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* keep the same position in the list if "rethrottle_thread" changes our throttle level and
|
||
|
|
* then puts us back to the original level before we get a chance to run
|
||
|
|
*/
|
||
|
|
if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED && ut->uu_on_throttlelist != mylevel) {
|
||
|
|
/*
|
||
|
|
* must have been awakened via "rethrottle_thread" (the timer pulls us off the list)
|
||
|
|
* and we've changed our throttling level, so pull ourselves off of the appropriate list
|
||
|
|
* and make sure we get put on the tail of the new list since we're starting anew w/r to
|
||
|
|
* the throttling engine
|
||
|
|
*/
|
||
|
|
TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
|
||
|
|
ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
|
||
|
|
insert_tail = TRUE;
|
||
|
|
}
|
||
|
|
if (ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED) {
|
||
|
|
if (throttle_add_to_list(info, ut, mylevel, insert_tail) == THROTTLE_LEVEL_END) {
|
||
|
|
goto done;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
assert(throttling_level >= THROTTLE_LEVEL_START && throttling_level <= THROTTLE_LEVEL_END);
|
||
|
|
|
||
|
|
s = ml_set_interrupts_enabled(FALSE);
|
||
|
|
lck_spin_lock(&ut->uu_rethrottle_lock);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* this is the critical section w/r to our interaction
|
||
|
|
* with "rethrottle_thread"
|
||
|
|
*/
|
||
|
|
if (ut->uu_was_rethrottled) {
|
||
|
|
lck_spin_unlock(&ut->uu_rethrottle_lock);
|
||
|
|
ml_set_interrupts_enabled(s);
|
||
|
|
lck_mtx_yield(&info->throttle_lock);
|
||
|
|
|
||
|
|
KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 103)),
|
||
|
|
uthread_tid(ut), ut->uu_on_throttlelist, 0, 0, 0);
|
||
|
|
|
||
|
|
ut->uu_was_rethrottled = false;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, PROCESS_THROTTLED)) | DBG_FUNC_NONE,
|
||
|
|
info->throttle_last_IO_pid[throttling_level], throttling_level, proc_selfpid(), mylevel, 0);
|
||
|
|
|
||
|
|
if (sleep_cnt == 0) {
|
||
|
|
KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START,
|
||
|
|
throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
|
||
|
|
throttled_count[mylevel]++;
|
||
|
|
}
|
||
|
|
ut->uu_wmesg = "throttle_lowpri_io";
|
||
|
|
|
||
|
|
assert_wait((caddr_t)&ut->uu_on_throttlelist, THREAD_UNINT);
|
||
|
|
|
||
|
|
ut->uu_is_throttled = true;
|
||
|
|
lck_spin_unlock(&ut->uu_rethrottle_lock);
|
||
|
|
ml_set_interrupts_enabled(s);
|
||
|
|
|
||
|
|
lck_mtx_unlock(&info->throttle_lock);
|
||
|
|
|
||
|
|
thread_block(THREAD_CONTINUE_NULL);
|
||
|
|
|
||
|
|
ut->uu_wmesg = NULL;
|
||
|
|
|
||
|
|
ut->uu_is_throttled = false;
|
||
|
|
ut->uu_was_rethrottled = false;
|
||
|
|
|
||
|
|
lck_mtx_lock(&info->throttle_lock);
|
||
|
|
|
||
|
|
sleep_cnt++;
|
||
|
|
|
||
|
|
if (sleep_amount == 0) {
|
||
|
|
insert_tail = FALSE;
|
||
|
|
} else if (info->throttle_io_period_num < throttle_io_period_num ||
|
||
|
|
(info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
|
||
|
|
insert_tail = FALSE;
|
||
|
|
sleep_amount = 0;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
done:
|
||
|
|
if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
|
||
|
|
TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
|
||
|
|
ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
|
||
|
|
}
|
||
|
|
lck_mtx_unlock(&info->throttle_lock);
|
||
|
|
|
||
|
|
if (sleep_cnt) {
|
||
|
|
KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END,
|
||
|
|
throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
|
||
|
|
/*
|
||
|
|
* We update the stats for the last pid which opened a throttle window for the throttled thread.
|
||
|
|
* This might not be completely accurate since the multiple throttles seen by the lower tier pid
|
||
|
|
* might have been caused by various higher prio pids. However, updating these stats accurately
|
||
|
|
* means doing a proc_find while holding the throttle lock which leads to deadlock.
|
||
|
|
*/
|
||
|
|
throttle_update_proc_stats(info->throttle_last_IO_pid[throttling_level], sleep_cnt);
|
||
|
|
}
|
||
|
|
|
||
|
|
ut->uu_throttle_info = NULL;
|
||
|
|
ut->uu_throttle_bc = false;
|
||
|
|
ut->uu_lowpri_window = 0;
|
||
|
|
|
||
|
|
throttle_info_rel(info);
|
||
|
|
|
||
|
|
return sleep_cnt;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* returns TRUE if the throttle_lowpri_io called with the same sleep_amount would've slept
|
||
|
|
* This function mimics the most of the throttle_lowpri_io checks but without actual sleeping
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
throttle_lowpri_io_will_be_throttled(int sleep_amount)
|
||
|
|
{
|
||
|
|
if (sleep_amount == 0) {
|
||
|
|
return FALSE;
|
||
|
|
}
|
||
|
|
|
||
|
|
uthread_t ut = current_uthread();
|
||
|
|
if (ut->uu_lowpri_window == 0) {
|
||
|
|
return FALSE;
|
||
|
|
}
|
||
|
|
|
||
|
|
struct _throttle_io_info_t *info = ut->uu_throttle_info;
|
||
|
|
if (info == NULL) {
|
||
|
|
return FALSE;
|
||
|
|
}
|
||
|
|
|
||
|
|
lck_mtx_lock(&info->throttle_lock);
|
||
|
|
assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
|
||
|
|
|
||
|
|
if (sleep_amount == 1 && !ut->uu_throttle_bc) {
|
||
|
|
sleep_amount = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
int result = FALSE;
|
||
|
|
|
||
|
|
int throttle_type = throttle_io_will_be_throttled_internal(info, NULL, NULL);
|
||
|
|
if (throttle_type > THROTTLE_DISENGAGED) {
|
||
|
|
result = TRUE;
|
||
|
|
if ((throttle_type == THROTTLE_ENGAGED) && (sleep_amount == 0)) {
|
||
|
|
result = FALSE;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
lck_mtx_unlock(&info->throttle_lock);
|
||
|
|
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* set a kernel thread's IO policy. policy can be:
|
||
|
|
* IOPOL_NORMAL, IOPOL_THROTTLE, IOPOL_PASSIVE, IOPOL_UTILITY, IOPOL_STANDARD
|
||
|
|
*
|
||
|
|
* explanations about these policies are in the man page of setiopolicy_np
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_set_thread_io_policy(int policy)
|
||
|
|
{
|
||
|
|
proc_set_thread_policy(current_thread(), TASK_POLICY_INTERNAL, TASK_POLICY_IOPOL, policy);
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
throttle_get_thread_effective_io_policy()
|
||
|
|
{
|
||
|
|
return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
throttle_thread_io_tier_above_metadata(void)
|
||
|
|
{
|
||
|
|
return throttle_get_thread_effective_io_policy() < IOSCHED_METADATA_TIER;
|
||
|
|
}
|
||
|
|
|
||
|
|
void
|
||
|
|
throttle_info_reset_window(uthread_t ut)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
|
||
|
|
if (ut == NULL) {
|
||
|
|
ut = current_uthread();
|
||
|
|
}
|
||
|
|
|
||
|
|
if ((info = ut->uu_throttle_info)) {
|
||
|
|
throttle_info_rel(info);
|
||
|
|
|
||
|
|
ut->uu_throttle_info = NULL;
|
||
|
|
ut->uu_lowpri_window = 0;
|
||
|
|
ut->uu_throttle_bc = false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static
|
||
|
|
void
|
||
|
|
throttle_info_set_initial_window(uthread_t ut, struct _throttle_io_info_t *info, boolean_t BC_throttle, boolean_t isssd)
|
||
|
|
{
|
||
|
|
if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (info->throttle_io_periods == 0) {
|
||
|
|
throttle_init_throttle_period(info, isssd);
|
||
|
|
}
|
||
|
|
if (ut->uu_throttle_info == NULL) {
|
||
|
|
ut->uu_throttle_info = info;
|
||
|
|
throttle_info_ref(info);
|
||
|
|
DEBUG_ALLOC_THROTTLE_INFO("updating info = %p\n", info, info );
|
||
|
|
|
||
|
|
ut->uu_lowpri_window = 1;
|
||
|
|
ut->uu_throttle_bc = BC_throttle;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Update inflight IO count and throttling window
|
||
|
|
* Should be called when an IO is done
|
||
|
|
*
|
||
|
|
* Only affects IO that was sent through spec_strategy
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_end_io(buf_t bp)
|
||
|
|
{
|
||
|
|
vnode_t vp;
|
||
|
|
mount_t mp;
|
||
|
|
struct bufattr *bap;
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
int io_tier;
|
||
|
|
|
||
|
|
bap = &bp->b_attr;
|
||
|
|
if (!ISSET(bap->ba_flags, BA_STRATEGY_TRACKED_IO)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
CLR(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
|
||
|
|
|
||
|
|
vp = buf_vnode(bp);
|
||
|
|
mp = vp->v_mount;
|
||
|
|
|
||
|
|
if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) {
|
||
|
|
info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
|
||
|
|
} else if (mp != NULL) {
|
||
|
|
info = &_throttle_io_info[mp->mnt_devbsdunit];
|
||
|
|
} else {
|
||
|
|
info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
|
||
|
|
}
|
||
|
|
|
||
|
|
io_tier = GET_BUFATTR_IO_TIER(bap);
|
||
|
|
if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
|
||
|
|
io_tier--;
|
||
|
|
}
|
||
|
|
|
||
|
|
throttle_info_end_io_internal(info, io_tier);
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Decrement inflight count initially incremented by throttle_info_update_internal
|
||
|
|
*/
|
||
|
|
static
|
||
|
|
void
|
||
|
|
throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level)
|
||
|
|
{
|
||
|
|
if (throttle_level == THROTTLE_LEVEL_NONE) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
microuptime(&info->throttle_window_start_timestamp[throttle_level]);
|
||
|
|
OSDecrementAtomic(&info->throttle_inflight_count[throttle_level]);
|
||
|
|
assert(info->throttle_inflight_count[throttle_level] >= 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* If inflight is TRUE and bap is NULL then the caller is responsible for calling
|
||
|
|
* throttle_info_end_io_internal to avoid leaking in-flight I/O.
|
||
|
|
*/
|
||
|
|
static
|
||
|
|
int
|
||
|
|
throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap)
|
||
|
|
{
|
||
|
|
int thread_throttle_level;
|
||
|
|
|
||
|
|
if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
|
||
|
|
return THROTTLE_LEVEL_NONE;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (ut == NULL) {
|
||
|
|
ut = current_uthread();
|
||
|
|
}
|
||
|
|
|
||
|
|
if (bap && inflight && !ut->uu_throttle_bc) {
|
||
|
|
thread_throttle_level = GET_BUFATTR_IO_TIER(bap);
|
||
|
|
if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
|
||
|
|
thread_throttle_level--;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
thread_throttle_level = throttle_get_thread_throttle_level(ut);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (thread_throttle_level != THROTTLE_LEVEL_NONE) {
|
||
|
|
if (!ISSET(flags, B_PASSIVE)) {
|
||
|
|
info->throttle_last_IO_pid[thread_throttle_level] = proc_selfpid();
|
||
|
|
if (inflight && !ut->uu_throttle_bc) {
|
||
|
|
if (NULL != bap) {
|
||
|
|
SET(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
|
||
|
|
}
|
||
|
|
OSIncrementAtomic(&info->throttle_inflight_count[thread_throttle_level]);
|
||
|
|
} else {
|
||
|
|
microuptime(&info->throttle_window_start_timestamp[thread_throttle_level]);
|
||
|
|
}
|
||
|
|
KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, OPEN_THROTTLE_WINDOW)) | DBG_FUNC_NONE,
|
||
|
|
proc_getpid(current_proc()), thread_throttle_level, 0, 0, 0);
|
||
|
|
}
|
||
|
|
microuptime(&info->throttle_last_IO_timestamp[thread_throttle_level]);
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
if (thread_throttle_level >= THROTTLE_LEVEL_THROTTLED) {
|
||
|
|
/*
|
||
|
|
* I'd really like to do the IOSleep here, but
|
||
|
|
* we may be holding all kinds of filesystem related locks
|
||
|
|
* and the pages for this I/O marked 'busy'...
|
||
|
|
* we don't want to cause a normal task to block on
|
||
|
|
* one of these locks while we're throttling a task marked
|
||
|
|
* for low priority I/O... we'll mark the uthread and
|
||
|
|
* do the delay just before we return from the system
|
||
|
|
* call that triggered this I/O or from vnode_pagein
|
||
|
|
*/
|
||
|
|
OSAddAtomic(1, &info->throttle_io_count);
|
||
|
|
|
||
|
|
throttle_info_set_initial_window(ut, info, FALSE, isssd);
|
||
|
|
}
|
||
|
|
|
||
|
|
return thread_throttle_level;
|
||
|
|
}
|
||
|
|
|
||
|
|
void *
|
||
|
|
throttle_info_update_by_mount(mount_t mp)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
uthread_t ut;
|
||
|
|
boolean_t isssd = FALSE;
|
||
|
|
|
||
|
|
ut = current_uthread();
|
||
|
|
|
||
|
|
if (mp != NULL) {
|
||
|
|
if (disk_conditioner_mount_is_ssd(mp)) {
|
||
|
|
isssd = TRUE;
|
||
|
|
}
|
||
|
|
info = &_throttle_io_info[mp->mnt_devbsdunit];
|
||
|
|
} else {
|
||
|
|
info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!ut->uu_lowpri_window) {
|
||
|
|
throttle_info_set_initial_window(ut, info, FALSE, isssd);
|
||
|
|
}
|
||
|
|
|
||
|
|
return info;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* this is usually called before every I/O, used for throttled I/O
|
||
|
|
* book keeping. This routine has low overhead and does not sleep
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_update(void *throttle_info, int flags)
|
||
|
|
{
|
||
|
|
if (throttle_info) {
|
||
|
|
throttle_info_update_internal(throttle_info, NULL, flags, FALSE, FALSE, NULL);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* this is usually called before every I/O, used for throttled I/O
|
||
|
|
* book keeping. This routine has low overhead and does not sleep
|
||
|
|
*/
|
||
|
|
void
|
||
|
|
throttle_info_update_by_mask(void *throttle_info_handle, int flags)
|
||
|
|
{
|
||
|
|
void *throttle_info = throttle_info_handle;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* for now we only use the lowest bit of the throttle mask, so the
|
||
|
|
* handle is the same as the throttle_info. Later if we store a
|
||
|
|
* set of throttle infos in the handle, we will want to loop through
|
||
|
|
* them and call throttle_info_update in a loop
|
||
|
|
*/
|
||
|
|
throttle_info_update(throttle_info, flags);
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* KPI routine
|
||
|
|
*
|
||
|
|
* This routine marks the throttle info as disabled. Used for mount points which
|
||
|
|
* support I/O scheduling.
|
||
|
|
*/
|
||
|
|
|
||
|
|
void
|
||
|
|
throttle_info_disable_throttle(int devno, boolean_t isfusion)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info;
|
||
|
|
|
||
|
|
if (devno < 0 || devno >= LOWPRI_MAX_NUM_DEV) {
|
||
|
|
panic("Illegal devno (%d) passed into throttle_info_disable_throttle()", devno);
|
||
|
|
}
|
||
|
|
|
||
|
|
info = &_throttle_io_info[devno];
|
||
|
|
// don't disable software throttling on devices that are part of a fusion device
|
||
|
|
// and override the software throttle periods to use HDD periods
|
||
|
|
if (isfusion) {
|
||
|
|
info->throttle_is_fusion_with_priority = isfusion;
|
||
|
|
throttle_init_throttle_period(info, FALSE);
|
||
|
|
}
|
||
|
|
info->throttle_disabled = !info->throttle_is_fusion_with_priority;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* KPI routine (private)
|
||
|
|
* Called to determine if this IO is being throttled to this level so that it can be treated specially
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
throttle_info_io_will_be_throttled(void * throttle_info, int policy)
|
||
|
|
{
|
||
|
|
struct _throttle_io_info_t *info = throttle_info;
|
||
|
|
struct timeval elapsed;
|
||
|
|
uint64_t elapsed_msecs;
|
||
|
|
int throttle_level;
|
||
|
|
int thread_throttle_level;
|
||
|
|
|
||
|
|
switch (policy) {
|
||
|
|
case IOPOL_THROTTLE:
|
||
|
|
thread_throttle_level = THROTTLE_LEVEL_TIER3;
|
||
|
|
break;
|
||
|
|
case IOPOL_UTILITY:
|
||
|
|
thread_throttle_level = THROTTLE_LEVEL_TIER2;
|
||
|
|
break;
|
||
|
|
case IOPOL_STANDARD:
|
||
|
|
thread_throttle_level = THROTTLE_LEVEL_TIER1;
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
thread_throttle_level = THROTTLE_LEVEL_TIER0;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
|
||
|
|
if (info->throttle_inflight_count[throttle_level]) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
microuptime(&elapsed);
|
||
|
|
timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
|
||
|
|
elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
|
||
|
|
|
||
|
|
if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (throttle_level >= thread_throttle_level) {
|
||
|
|
/*
|
||
|
|
* we're beyond all of the throttle windows
|
||
|
|
* so go ahead and treat as normal I/O
|
||
|
|
*/
|
||
|
|
return THROTTLE_DISENGAGED;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* we're in the throttle window
|
||
|
|
*/
|
||
|
|
return THROTTLE_ENGAGED;
|
||
|
|
}
|
||
|
|
|
||
|
|
int
|
||
|
|
throttle_lowpri_window(void)
|
||
|
|
{
|
||
|
|
return current_uthread()->uu_lowpri_window;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
#if CONFIG_IOSCHED
|
||
|
|
int upl_get_cached_tier(void *);
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#if CONFIG_PHYS_WRITE_ACCT
|
||
|
|
extern thread_t pm_sync_thread;
|
||
|
|
#endif /* CONFIG_PHYS_WRITE_ACCT */
|
||
|
|
|
||
|
|
int
|
||
|
|
spec_strategy(struct vnop_strategy_args *ap)
|
||
|
|
{
|
||
|
|
buf_t bp;
|
||
|
|
int bflags;
|
||
|
|
int io_tier;
|
||
|
|
int passive;
|
||
|
|
dev_t bdev;
|
||
|
|
uthread_t ut;
|
||
|
|
vnode_t vp;
|
||
|
|
mount_t mp;
|
||
|
|
struct bufattr *bap;
|
||
|
|
int strategy_ret;
|
||
|
|
struct _throttle_io_info_t *throttle_info;
|
||
|
|
boolean_t isssd = FALSE;
|
||
|
|
boolean_t inflight = FALSE;
|
||
|
|
boolean_t upgrade = FALSE;
|
||
|
|
int code = 0;
|
||
|
|
|
||
|
|
#if CONFIG_DELAY_IDLE_SLEEP
|
||
|
|
proc_t curproc = current_proc();
|
||
|
|
#endif /* CONFIG_DELAY_IDLE_SLEEP */
|
||
|
|
|
||
|
|
bp = ap->a_bp;
|
||
|
|
bdev = buf_device(bp);
|
||
|
|
vp = buf_vnode(bp);
|
||
|
|
mp = vp ? vp->v_mount : NULL;
|
||
|
|
bap = &bp->b_attr;
|
||
|
|
|
||
|
|
#if CONFIG_PHYS_WRITE_ACCT
|
||
|
|
if (current_thread() == pm_sync_thread) {
|
||
|
|
OSAddAtomic64(buf_count(bp), (SInt64 *)&(kernel_pm_writes));
|
||
|
|
}
|
||
|
|
#endif /* CONFIG_PHYS_WRITE_ACCT */
|
||
|
|
|
||
|
|
#if CONFIG_IOSCHED
|
||
|
|
if (bp->b_flags & B_CLUSTER) {
|
||
|
|
io_tier = upl_get_cached_tier(bp->b_upl);
|
||
|
|
|
||
|
|
if (io_tier == -1) {
|
||
|
|
io_tier = throttle_get_io_policy(&ut);
|
||
|
|
}
|
||
|
|
#if DEVELOPMENT || DEBUG
|
||
|
|
else {
|
||
|
|
int my_io_tier = throttle_get_io_policy(&ut);
|
||
|
|
|
||
|
|
if (io_tier != my_io_tier) {
|
||
|
|
KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
} else {
|
||
|
|
io_tier = throttle_get_io_policy(&ut);
|
||
|
|
}
|
||
|
|
#else
|
||
|
|
io_tier = throttle_get_io_policy(&ut);
|
||
|
|
#endif
|
||
|
|
passive = throttle_get_passive_io_policy(&ut);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Mark if the I/O was upgraded by throttle_get_thread_throttle_level
|
||
|
|
* while preserving the original issued tier (throttle_get_io_policy
|
||
|
|
* does not return upgraded tiers)
|
||
|
|
*/
|
||
|
|
if (mp && io_tier > throttle_get_thread_throttle_level_internal(ut, io_tier)) {
|
||
|
|
#if CONFIG_IOSCHED
|
||
|
|
if (!(mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) {
|
||
|
|
upgrade = TRUE;
|
||
|
|
}
|
||
|
|
#else /* CONFIG_IOSCHED */
|
||
|
|
upgrade = TRUE;
|
||
|
|
#endif /* CONFIG_IOSCHED */
|
||
|
|
}
|
||
|
|
|
||
|
|
if (bp->b_flags & B_META) {
|
||
|
|
bap->ba_flags |= BA_META;
|
||
|
|
}
|
||
|
|
|
||
|
|
#if CONFIG_IOSCHED
|
||
|
|
/*
|
||
|
|
* For metadata reads, ceil the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited, otherwise
|
||
|
|
* ceil it to IOSCHED_METADATA_TIER. Mark them passive if the I/O tier was upgraded.
|
||
|
|
* For metadata writes, set the I/O tier to IOSCHED_METADATA_EXPEDITED_TIER if they are expedited. Otherwise
|
||
|
|
* set it to IOSCHED_METADATA_TIER. In addition, mark them as passive.
|
||
|
|
*/
|
||
|
|
if (bap->ba_flags & BA_META) {
|
||
|
|
if ((mp && (mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) || (bap->ba_flags & BA_IO_SCHEDULED)) {
|
||
|
|
if (bp->b_flags & B_READ) {
|
||
|
|
if ((bap->ba_flags & BA_EXPEDITED_META_IO) && (io_tier > IOSCHED_METADATA_EXPEDITED_TIER)) {
|
||
|
|
io_tier = IOSCHED_METADATA_EXPEDITED_TIER;
|
||
|
|
passive = 1;
|
||
|
|
} else if (io_tier > IOSCHED_METADATA_TIER) {
|
||
|
|
io_tier = IOSCHED_METADATA_TIER;
|
||
|
|
passive = 1;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
if (bap->ba_flags & BA_EXPEDITED_META_IO) {
|
||
|
|
io_tier = IOSCHED_METADATA_EXPEDITED_TIER;
|
||
|
|
} else {
|
||
|
|
io_tier = IOSCHED_METADATA_TIER;
|
||
|
|
}
|
||
|
|
passive = 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
#endif /* CONFIG_IOSCHED */
|
||
|
|
|
||
|
|
SET_BUFATTR_IO_TIER(bap, io_tier);
|
||
|
|
|
||
|
|
if (passive) {
|
||
|
|
bp->b_flags |= B_PASSIVE;
|
||
|
|
bap->ba_flags |= BA_PASSIVE;
|
||
|
|
}
|
||
|
|
|
||
|
|
#if CONFIG_DELAY_IDLE_SLEEP
|
||
|
|
if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP)) {
|
||
|
|
bap->ba_flags |= BA_DELAYIDLESLEEP;
|
||
|
|
}
|
||
|
|
#endif /* CONFIG_DELAY_IDLE_SLEEP */
|
||
|
|
|
||
|
|
bflags = bp->b_flags;
|
||
|
|
|
||
|
|
if (((bflags & B_READ) == 0) && ((bflags & B_ASYNC) == 0)) {
|
||
|
|
bufattr_markquickcomplete(bap);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (bflags & B_READ) {
|
||
|
|
code |= DKIO_READ;
|
||
|
|
}
|
||
|
|
if (bflags & B_ASYNC) {
|
||
|
|
code |= DKIO_ASYNC;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (bap->ba_flags & BA_META) {
|
||
|
|
code |= DKIO_META;
|
||
|
|
} else if (bflags & B_PAGEIO) {
|
||
|
|
code |= DKIO_PAGING;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (io_tier != 0) {
|
||
|
|
code |= DKIO_THROTTLE;
|
||
|
|
}
|
||
|
|
|
||
|
|
code |= ((io_tier << DKIO_TIER_SHIFT) & DKIO_TIER_MASK);
|
||
|
|
|
||
|
|
if (bflags & B_PASSIVE) {
|
||
|
|
code |= DKIO_PASSIVE;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (bap->ba_flags & BA_NOCACHE) {
|
||
|
|
code |= DKIO_NOCACHE;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (upgrade) {
|
||
|
|
code |= DKIO_TIER_UPGRADE;
|
||
|
|
SET(bap->ba_flags, BA_IO_TIER_UPGRADE);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (kdebug_enable) {
|
||
|
|
KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, FSDBG_CODE(DBG_DKRW, code) | DBG_FUNC_NONE,
|
||
|
|
buf_kernel_addrperm_addr(bp), bdev, buf_blkno(bp), buf_count(bp), 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
#if CONFIG_IO_COMPRESSION_STATS
|
||
|
|
// Do not run IO Compression Stats when a privilege thread is active
|
||
|
|
if (!is_vm_privileged() && !is_external_pageout_thread()) {
|
||
|
|
io_compression_stats(bp);
|
||
|
|
}
|
||
|
|
#endif /* CONFIG_IO_COMPRESSION_STATS */
|
||
|
|
thread_update_io_stats(current_thread(), buf_count(bp), code);
|
||
|
|
|
||
|
|
if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) {
|
||
|
|
if (!vp->v_un.vu_specinfo->si_initted) {
|
||
|
|
SPEC_INIT_BSDUNIT(vp, vfs_context_current());
|
||
|
|
}
|
||
|
|
if (vp->v_un.vu_specinfo->si_devbsdunit > (LOWPRI_MAX_NUM_DEV - 1)) {
|
||
|
|
panic("Invalid value (%d) for si_devbsdunit for vnode %p",
|
||
|
|
vp->v_un.vu_specinfo->si_devbsdunit, vp);
|
||
|
|
}
|
||
|
|
if (vp->v_un.vu_specinfo->si_isssd > 1) {
|
||
|
|
panic("Invalid value (%d) for si_isssd for vnode %p",
|
||
|
|
vp->v_un.vu_specinfo->si_isssd, vp);
|
||
|
|
}
|
||
|
|
throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
|
||
|
|
isssd = vp->v_un.vu_specinfo->si_isssd;
|
||
|
|
} else if (mp != NULL) {
|
||
|
|
if (disk_conditioner_mount_is_ssd(mp)) {
|
||
|
|
isssd = TRUE;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* Partially initialized mounts don't have a final devbsdunit and should not be tracked.
|
||
|
|
* Verify that devbsdunit is initialized (non-zero) or that 0 is the correct initialized value
|
||
|
|
* (mnt_throttle_mask is initialized and num_trailing_0 would be 0)
|
||
|
|
*/
|
||
|
|
if (mp->mnt_devbsdunit || (mp->mnt_throttle_mask != LOWPRI_MAX_NUM_DEV - 1 && mp->mnt_throttle_mask & 0x1)) {
|
||
|
|
inflight = TRUE;
|
||
|
|
}
|
||
|
|
throttle_info = &_throttle_io_info[mp->mnt_devbsdunit];
|
||
|
|
} else {
|
||
|
|
throttle_info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
|
||
|
|
}
|
||
|
|
|
||
|
|
throttle_info_update_internal(throttle_info, ut, bflags, isssd, inflight, bap);
|
||
|
|
|
||
|
|
if ((bflags & B_READ) == 0) {
|
||
|
|
microuptime(&throttle_info->throttle_last_write_timestamp);
|
||
|
|
|
||
|
|
if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) {
|
||
|
|
mp->mnt_last_write_issued_timestamp = throttle_info->throttle_last_write_timestamp;
|
||
|
|
INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_write_size);
|
||
|
|
}
|
||
|
|
} else if (!(vp && (vp->v_type == VBLK || vp->v_type == VCHR)) && mp) {
|
||
|
|
INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_read_size);
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* The BootCache may give us special information about
|
||
|
|
* the IO, so it returns special values that we check
|
||
|
|
* for here.
|
||
|
|
*
|
||
|
|
* IO_SATISFIED_BY_CACHE
|
||
|
|
* The read has been satisfied by the boot cache. Don't
|
||
|
|
* throttle the thread unnecessarily.
|
||
|
|
*
|
||
|
|
* IO_SHOULD_BE_THROTTLED
|
||
|
|
* The boot cache is playing back a playlist and this IO
|
||
|
|
* cut through. Throttle it so we're not cutting through
|
||
|
|
* the boot cache too often.
|
||
|
|
*
|
||
|
|
* Note that typical strategy routines are defined with
|
||
|
|
* a void return so we'll get garbage here. In the
|
||
|
|
* unlikely case the garbage matches our special return
|
||
|
|
* value, it's not a big deal since we're only adjusting
|
||
|
|
* the throttling delay.
|
||
|
|
*/
|
||
|
|
#define IO_SATISFIED_BY_CACHE ((int)0xcafefeed)
|
||
|
|
#define IO_SHOULD_BE_THROTTLED ((int)0xcafebeef)
|
||
|
|
#pragma clang diagnostic push
|
||
|
|
#pragma clang diagnostic ignored "-Wcast-function-type"
|
||
|
|
|
||
|
|
typedef int strategy_fcn_ret_t(struct buf *bp);
|
||
|
|
|
||
|
|
strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp);
|
||
|
|
|
||
|
|
#pragma clang diagnostic pop
|
||
|
|
|
||
|
|
// disk conditioner needs to track when this I/O actually starts
|
||
|
|
// which means track it after `strategy` which may include delays
|
||
|
|
// from inflight I/Os
|
||
|
|
microuptime(&bp->b_timestamp_tv);
|
||
|
|
|
||
|
|
if (IO_SATISFIED_BY_CACHE == strategy_ret) {
|
||
|
|
/*
|
||
|
|
* If this was a throttled IO satisfied by the boot cache,
|
||
|
|
* don't delay the thread.
|
||
|
|
*/
|
||
|
|
throttle_info_reset_window(ut);
|
||
|
|
} else if (IO_SHOULD_BE_THROTTLED == strategy_ret) {
|
||
|
|
/*
|
||
|
|
* If the boot cache indicates this IO should be throttled,
|
||
|
|
* delay the thread.
|
||
|
|
*/
|
||
|
|
throttle_info_set_initial_window(ut, throttle_info, TRUE, isssd);
|
||
|
|
}
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* This is a noop, simply returning what one has been given.
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_blockmap(__unused struct vnop_blockmap_args *ap)
|
||
|
|
{
|
||
|
|
return ENOTSUP;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Device close routine
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_close(struct vnop_close_args *ap)
|
||
|
|
{
|
||
|
|
struct vnode *vp = ap->a_vp;
|
||
|
|
dev_t dev = vp->v_rdev;
|
||
|
|
int error = 0;
|
||
|
|
int flags = ap->a_fflag;
|
||
|
|
struct proc *p = vfs_context_proc(ap->a_context);
|
||
|
|
struct session *sessp;
|
||
|
|
struct pgrp *pg;
|
||
|
|
|
||
|
|
switch (vp->v_type) {
|
||
|
|
case VCHR:
|
||
|
|
/*
|
||
|
|
* Hack: a tty device that is a controlling terminal
|
||
|
|
* has a reference from the session structure.
|
||
|
|
* We cannot easily tell that a character device is
|
||
|
|
* a controlling terminal, unless it is the closing
|
||
|
|
* process' controlling terminal. In that case,
|
||
|
|
* if the reference count is 1 (this is the very
|
||
|
|
* last close)
|
||
|
|
*/
|
||
|
|
pg = proc_pgrp(p, &sessp);
|
||
|
|
devsw_lock(dev, S_IFCHR);
|
||
|
|
if (sessp != SESSION_NULL) {
|
||
|
|
if (vp == sessp->s_ttyvp && vcount(vp) == 1) {
|
||
|
|
struct tty *tp = TTY_NULL;
|
||
|
|
|
||
|
|
devsw_unlock(dev, S_IFCHR);
|
||
|
|
session_lock(sessp);
|
||
|
|
if (vp == sessp->s_ttyvp) {
|
||
|
|
tp = session_clear_tty_locked(sessp);
|
||
|
|
}
|
||
|
|
session_unlock(sessp);
|
||
|
|
|
||
|
|
if (tp != TTY_NULL) {
|
||
|
|
ttyfree(tp);
|
||
|
|
}
|
||
|
|
devsw_lock(dev, S_IFCHR);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
pgrp_rele(pg);
|
||
|
|
|
||
|
|
if (--vp->v_specinfo->si_opencount < 0) {
|
||
|
|
panic("negative open count (c, %u, %u)", major(dev), minor(dev));
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* close on last reference or on vnode revoke call
|
||
|
|
*/
|
||
|
|
if (vcount(vp) == 0 || (flags & IO_REVOKE) != 0) {
|
||
|
|
error = cdevsw[major(dev)].d_close(dev, flags, S_IFCHR, p);
|
||
|
|
}
|
||
|
|
|
||
|
|
devsw_unlock(dev, S_IFCHR);
|
||
|
|
break;
|
||
|
|
|
||
|
|
case VBLK:
|
||
|
|
/*
|
||
|
|
* If there is more than one outstanding open, don't
|
||
|
|
* send the close to the device.
|
||
|
|
*/
|
||
|
|
devsw_lock(dev, S_IFBLK);
|
||
|
|
if (vcount(vp) > 1) {
|
||
|
|
vp->v_specinfo->si_opencount--;
|
||
|
|
devsw_unlock(dev, S_IFBLK);
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
devsw_unlock(dev, S_IFBLK);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* On last close of a block device (that isn't mounted)
|
||
|
|
* we must invalidate any in core blocks, so that
|
||
|
|
* we can, for instance, change floppy disks.
|
||
|
|
*/
|
||
|
|
if ((error = spec_fsync_internal(vp, MNT_WAIT, ap->a_context))) {
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
|
||
|
|
error = buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0);
|
||
|
|
if (error) {
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
|
||
|
|
devsw_lock(dev, S_IFBLK);
|
||
|
|
|
||
|
|
if (--vp->v_specinfo->si_opencount < 0) {
|
||
|
|
panic("negative open count (b, %u, %u)", major(dev), minor(dev));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (vcount(vp) == 0) {
|
||
|
|
error = bdevsw[major(dev)].d_close(dev, flags, S_IFBLK, p);
|
||
|
|
}
|
||
|
|
|
||
|
|
devsw_unlock(dev, S_IFBLK);
|
||
|
|
break;
|
||
|
|
|
||
|
|
default:
|
||
|
|
panic("spec_close: not special");
|
||
|
|
return EBADF;
|
||
|
|
}
|
||
|
|
|
||
|
|
return error;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Return POSIX pathconf information applicable to special devices.
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_pathconf(struct vnop_pathconf_args *ap)
|
||
|
|
{
|
||
|
|
switch (ap->a_name) {
|
||
|
|
case _PC_LINK_MAX:
|
||
|
|
*ap->a_retval = LINK_MAX;
|
||
|
|
return 0;
|
||
|
|
case _PC_MAX_CANON:
|
||
|
|
*ap->a_retval = MAX_CANON;
|
||
|
|
return 0;
|
||
|
|
case _PC_MAX_INPUT:
|
||
|
|
*ap->a_retval = MAX_INPUT;
|
||
|
|
return 0;
|
||
|
|
case _PC_PIPE_BUF:
|
||
|
|
*ap->a_retval = PIPE_BUF;
|
||
|
|
return 0;
|
||
|
|
case _PC_CHOWN_RESTRICTED:
|
||
|
|
*ap->a_retval = 200112; /* _POSIX_CHOWN_RESTRICTED */
|
||
|
|
return 0;
|
||
|
|
case _PC_VDISABLE:
|
||
|
|
*ap->a_retval = _POSIX_VDISABLE;
|
||
|
|
return 0;
|
||
|
|
default:
|
||
|
|
return EINVAL;
|
||
|
|
}
|
||
|
|
/* NOTREACHED */
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Special device failed operation
|
||
|
|
*/
|
||
|
|
int
|
||
|
|
spec_ebadf(__unused void *dummy)
|
||
|
|
{
|
||
|
|
return EBADF;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Blktooff derives file offset from logical block number */
|
||
|
|
int
|
||
|
|
spec_blktooff(struct vnop_blktooff_args *ap)
|
||
|
|
{
|
||
|
|
struct vnode *vp = ap->a_vp;
|
||
|
|
|
||
|
|
switch (vp->v_type) {
|
||
|
|
case VCHR:
|
||
|
|
*ap->a_offset = (off_t)-1; /* failure */
|
||
|
|
return ENOTSUP;
|
||
|
|
|
||
|
|
case VBLK:
|
||
|
|
printf("spec_blktooff: not implemented for VBLK\n");
|
||
|
|
*ap->a_offset = (off_t)-1; /* failure */
|
||
|
|
return ENOTSUP;
|
||
|
|
|
||
|
|
default:
|
||
|
|
panic("spec_blktooff type");
|
||
|
|
}
|
||
|
|
/* NOTREACHED */
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Offtoblk derives logical block number from file offset */
|
||
|
|
int
|
||
|
|
spec_offtoblk(struct vnop_offtoblk_args *ap)
|
||
|
|
{
|
||
|
|
struct vnode *vp = ap->a_vp;
|
||
|
|
|
||
|
|
switch (vp->v_type) {
|
||
|
|
case VCHR:
|
||
|
|
*ap->a_lblkno = (daddr64_t)-1; /* failure */
|
||
|
|
return ENOTSUP;
|
||
|
|
|
||
|
|
case VBLK:
|
||
|
|
printf("spec_offtoblk: not implemented for VBLK\n");
|
||
|
|
*ap->a_lblkno = (daddr64_t)-1; /* failure */
|
||
|
|
return ENOTSUP;
|
||
|
|
|
||
|
|
default:
|
||
|
|
panic("spec_offtoblk type");
|
||
|
|
}
|
||
|
|
/* NOTREACHED */
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int filt_specattach(struct knote *kn, struct kevent_qos_s *kev);
|
||
|
|
static void filt_specdetach(struct knote *kn);
|
||
|
|
static int filt_specevent(struct knote *kn, long hint);
|
||
|
|
static int filt_spectouch(struct knote *kn, struct kevent_qos_s *kev);
|
||
|
|
static int filt_specprocess(struct knote *kn, struct kevent_qos_s *kev);
|
||
|
|
|
||
|
|
SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = {
|
||
|
|
.f_isfd = 1,
|
||
|
|
.f_attach = filt_specattach,
|
||
|
|
.f_detach = filt_specdetach,
|
||
|
|
.f_event = filt_specevent,
|
||
|
|
.f_touch = filt_spectouch,
|
||
|
|
.f_process = filt_specprocess,
|
||
|
|
};
|
||
|
|
|
||
|
|
static void
|
||
|
|
filt_spec_make_eof(struct knote *kn)
|
||
|
|
{
|
||
|
|
/*
|
||
|
|
* The spec filter might touch kn_flags from f_event
|
||
|
|
* without holding "the primitive lock", so make it atomic.
|
||
|
|
*/
|
||
|
|
os_atomic_or(&kn->kn_flags, EV_EOF | EV_ONESHOT, relaxed);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
filt_spec_common(struct knote *kn, struct kevent_qos_s *kev, bool attach)
|
||
|
|
{
|
||
|
|
uthread_t uth = current_uthread();
|
||
|
|
vfs_context_t ctx = vfs_context_current();
|
||
|
|
vnode_t vp = (vnode_t)fp_get_data(kn->kn_fp);
|
||
|
|
__block bool selrecorded = false;
|
||
|
|
struct select_set *old_wqs;
|
||
|
|
int64_t data = 0;
|
||
|
|
int ret, selret;
|
||
|
|
|
||
|
|
if (kn->kn_flags & EV_EOF) {
|
||
|
|
ret = FILTER_ACTIVE;
|
||
|
|
goto out;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!attach && vnode_getwithvid(vp, vnode_vid(vp)) != 0) {
|
||
|
|
filt_spec_make_eof(kn);
|
||
|
|
ret = FILTER_ACTIVE;
|
||
|
|
goto out;
|
||
|
|
}
|
||
|
|
|
||
|
|
selspec_record_hook_t cb = ^(struct selinfo *si) {
|
||
|
|
selspec_attach(kn, si);
|
||
|
|
selrecorded = true;
|
||
|
|
};
|
||
|
|
|
||
|
|
old_wqs = uth->uu_selset;
|
||
|
|
uth->uu_selset = SELSPEC_RECORD_MARKER;
|
||
|
|
selret = VNOP_SELECT(vp, knote_get_seltype(kn), 0, cb, ctx);
|
||
|
|
uth->uu_selset = old_wqs;
|
||
|
|
|
||
|
|
if (!attach) {
|
||
|
|
vnode_put(vp);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!selrecorded && selret == 0) {
|
||
|
|
/*
|
||
|
|
* The device indicated that there's no data to read,
|
||
|
|
* but didn't call `selrecord`.
|
||
|
|
*
|
||
|
|
* Nothing will be notified of changes to this vnode,
|
||
|
|
* so return an error back to user space on attach,
|
||
|
|
* or pretend the knote disappeared for other cases,
|
||
|
|
* to make it clear that the knote is not attached.
|
||
|
|
*/
|
||
|
|
if (attach) {
|
||
|
|
knote_set_error(kn, ENODEV);
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
filt_spec_make_eof(kn);
|
||
|
|
ret = FILTER_ACTIVE;
|
||
|
|
goto out;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (kn->kn_vnode_use_ofst) {
|
||
|
|
if (kn->kn_fp->fp_glob->fg_offset >= (uint32_t)selret) {
|
||
|
|
data = 0;
|
||
|
|
} else {
|
||
|
|
data = ((uint32_t)selret) - kn->kn_fp->fp_glob->fg_offset;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
data = selret;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (data >= knote_low_watermark(kn)) {
|
||
|
|
ret = FILTER_ACTIVE;
|
||
|
|
} else {
|
||
|
|
ret = 0;
|
||
|
|
}
|
||
|
|
out:
|
||
|
|
if (ret) {
|
||
|
|
knote_fill_kevent(kn, kev, data);
|
||
|
|
}
|
||
|
|
return ret;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
filt_specattach(struct knote *kn, __unused struct kevent_qos_s *kev)
|
||
|
|
{
|
||
|
|
vnode_t vp = (vnode_t)fp_get_data(kn->kn_fp); /* Already have iocount, and vnode is alive */
|
||
|
|
dev_t dev;
|
||
|
|
|
||
|
|
assert(vnode_ischr(vp));
|
||
|
|
|
||
|
|
dev = vnode_specrdev(vp);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* For a few special kinds of devices, we can attach knotes with
|
||
|
|
* no restrictions because their "select" vectors return the amount
|
||
|
|
* of data available. Others require an explicit NOTE_LOWAT with
|
||
|
|
* data of 1, indicating that the caller doesn't care about actual
|
||
|
|
* data counts, just an indication that the device has data.
|
||
|
|
*/
|
||
|
|
if (!kn->kn_vnode_kqok &&
|
||
|
|
((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) {
|
||
|
|
knote_set_error(kn, EINVAL);
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
return filt_spec_common(kn, kev, true);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void
|
||
|
|
filt_specdetach(struct knote *kn)
|
||
|
|
{
|
||
|
|
selspec_detach(kn);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
filt_specevent(struct knote *kn, long hint)
|
||
|
|
{
|
||
|
|
/* Due to selwakeup_internal() on SI_SELSPEC */
|
||
|
|
assert(KNOTE_IS_AUTODETACHED(kn));
|
||
|
|
knote_kn_hook_set_raw(kn, NULL);
|
||
|
|
|
||
|
|
/* called by selwakeup with the selspec_lock lock held */
|
||
|
|
if (hint & NOTE_REVOKE) {
|
||
|
|
filt_spec_make_eof(kn);
|
||
|
|
}
|
||
|
|
return FILTER_ACTIVE;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
filt_spectouch(struct knote *kn, struct kevent_qos_s *kev)
|
||
|
|
{
|
||
|
|
kn->kn_sdata = kev->data;
|
||
|
|
kn->kn_sfflags = kev->fflags;
|
||
|
|
|
||
|
|
return filt_spec_common(kn, kev, false);
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
filt_specprocess(struct knote *kn, struct kevent_qos_s *kev)
|
||
|
|
{
|
||
|
|
return filt_spec_common(kn, kev, false);
|
||
|
|
}
|