/* * Copyright (c) 2000-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ /*- * Copyright (c) 1994 Christopher G. Demetriou * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 */ /* * Some references: * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) * Leffler, et al.: The Design and Implementation of the 4.3BSD * UNIX Operating System (Addison Welley, 1989) */ #include #include #include #include #include #include #include #include #include #include #include #include #if DIAGNOSTIC #include #endif /* DIAGNOSTIC */ #include #include #include #include #include /* fslog_io_error() */ #include /* dk_error_description_t */ #include #include #include /* thread_block() */ #include #include #include #include #include #include #include int bcleanbuf(buf_t bp, boolean_t discard); static int brecover_data(buf_t bp); static boolean_t incore(vnode_t vp, daddr64_t blkno); /* timeout is in msecs */ static buf_t getnewbuf(int slpflag, int slptimeo, int *queue); static void bremfree_locked(buf_t bp); static void buf_reassign(buf_t bp, vnode_t newvp); static errno_t buf_acquire_locked(buf_t bp, int flags, int slpflag, int slptimeo); static int buf_iterprepare(vnode_t vp, struct buflists *, int flags); static void buf_itercomplete(vnode_t vp, struct buflists *, int flags); static boolean_t buffer_cache_gc(int); static buf_t buf_brelse_shadow(buf_t bp); static void buf_free_meta_store(buf_t bp); static buf_t buf_create_shadow_internal(buf_t bp, boolean_t force_copy, uintptr_t external_storage, void (*iodone)(buf_t, void *), void *arg, int priv); int bdwrite_internal(buf_t, int); extern void disk_conditioner_delay(buf_t, int, int, uint64_t); /* zone allocated buffer headers */ static void bcleanbuf_thread_init(void); static void bcleanbuf_thread(void); static ZONE_DEFINE_TYPE(buf_hdr_zone, "buf headers", struct buf, ZC_NONE); static int buf_hdr_count; /* * Definitions for the buffer hash lists. */ #define BUFHASH(dvp, lbn) \ (&bufhashtbl[((long)(dvp) / sizeof(*(dvp)) + (int)(lbn)) & bufhash]) LIST_HEAD(bufhashhdr, buf) * bufhashtbl, invalhash; u_long bufhash; static buf_t incore_locked(vnode_t vp, daddr64_t blkno, struct bufhashhdr *dp); /* Definitions for the buffer stats. */ struct bufstats bufstats; /* Number of delayed write buffers */ long nbdwrite = 0; int blaundrycnt = 0; static int boot_nbuf_headers = 0; static TAILQ_HEAD(delayqueue, buf) delaybufqueue; static TAILQ_HEAD(ioqueue, buf) iobufqueue; static TAILQ_HEAD(bqueues, buf) bufqueues[BQUEUES]; static int needbuffer; static int need_iobuffer; static LCK_GRP_DECLARE(buf_mtx_grp, "buffer cache"); static LCK_ATTR_DECLARE(buf_mtx_attr, 0, 0); static LCK_MTX_DECLARE_ATTR(iobuffer_mtxp, &buf_mtx_grp, &buf_mtx_attr); static LCK_MTX_DECLARE_ATTR(buf_mtx, &buf_mtx_grp, &buf_mtx_attr); static LCK_MTX_DECLARE_ATTR(buf_gc_callout, &buf_mtx_grp, &buf_mtx_attr); static uint32_t buf_busycount; #define FS_BUFFER_CACHE_GC_CALLOUTS_MAX_SIZE 16 typedef struct { void (* callout)(int, void *); void *context; } fs_buffer_cache_gc_callout_t; fs_buffer_cache_gc_callout_t fs_callouts[FS_BUFFER_CACHE_GC_CALLOUTS_MAX_SIZE] = { {NULL, NULL} }; static __inline__ int buf_timestamp(void) { struct timeval t; microuptime(&t); return (int)t.tv_sec; } /* * Insq/Remq for the buffer free lists. */ #define binsheadfree(bp, dp, whichq) do { \ TAILQ_INSERT_HEAD(dp, bp, b_freelist); \ } while (0) #define binstailfree(bp, dp, whichq) do { \ TAILQ_INSERT_TAIL(dp, bp, b_freelist); \ } while (0) #define BHASHENTCHECK(bp) \ if ((bp)->b_hash.le_prev != (struct buf **)0xdeadbeef) \ panic("%p: b_hash.le_prev is not deadbeef", (bp)); #define BLISTNONE(bp) \ (bp)->b_hash.le_next = (struct buf *)0; \ (bp)->b_hash.le_prev = (struct buf **)0xdeadbeef; /* * Insq/Remq for the vnode usage lists. */ #define bufinsvn(bp, dp) LIST_INSERT_HEAD(dp, bp, b_vnbufs) #define bufremvn(bp) { \ LIST_REMOVE(bp, b_vnbufs); \ (bp)->b_vnbufs.le_next = NOLIST; \ } /* * Time in seconds before a buffer on a list is * considered as a stale buffer */ #define LRU_IS_STALE 120 /* default value for the LRU */ #define AGE_IS_STALE 60 /* default value for the AGE */ #define META_IS_STALE 180 /* default value for the BQ_META */ int lru_is_stale = LRU_IS_STALE; int age_is_stale = AGE_IS_STALE; int meta_is_stale = META_IS_STALE; #define MAXLAUNDRY 10 /* LIST_INSERT_HEAD() with assertions */ static __inline__ void blistenterhead(struct bufhashhdr * head, buf_t bp) { if ((bp->b_hash.le_next = (head)->lh_first) != NULL) { (head)->lh_first->b_hash.le_prev = &(bp)->b_hash.le_next; } (head)->lh_first = bp; bp->b_hash.le_prev = &(head)->lh_first; if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef) { panic("blistenterhead: le_prev is deadbeef"); } } static __inline__ void binshash(buf_t bp, struct bufhashhdr *dp) { #if DIAGNOSTIC buf_t nbp; #endif /* DIAGNOSTIC */ BHASHENTCHECK(bp); #if DIAGNOSTIC nbp = dp->lh_first; for (; nbp != NULL; nbp = nbp->b_hash.le_next) { if (nbp == bp) { panic("buf already in hashlist"); } } #endif /* DIAGNOSTIC */ blistenterhead(dp, bp); } static __inline__ void bremhash(buf_t bp) { if (bp->b_hash.le_prev == (struct buf **)0xdeadbeef) { panic("bremhash le_prev is deadbeef"); } if (bp->b_hash.le_next == bp) { panic("bremhash: next points to self"); } if (bp->b_hash.le_next != NULL) { bp->b_hash.le_next->b_hash.le_prev = bp->b_hash.le_prev; } *bp->b_hash.le_prev = (bp)->b_hash.le_next; } /* * buf_mtx held. */ static __inline__ void bmovelaundry(buf_t bp) { bp->b_whichq = BQ_LAUNDRY; bp->b_timestamp = buf_timestamp(); binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY); blaundrycnt++; } static __inline__ void buf_release_credentials(buf_t bp) { if (IS_VALID_CRED(bp->b_rcred)) { kauth_cred_unref(&bp->b_rcred); } if (IS_VALID_CRED(bp->b_wcred)) { kauth_cred_unref(&bp->b_wcred); } } int buf_valid(buf_t bp) { if ((bp->b_flags & (B_DONE | B_DELWRI))) { return 1; } return 0; } int buf_fromcache(buf_t bp) { if ((bp->b_flags & B_CACHE)) { return 1; } return 0; } void buf_markinvalid(buf_t bp) { SET(bp->b_flags, B_INVAL); } void buf_markdelayed(buf_t bp) { if (!ISSET(bp->b_flags, B_DELWRI)) { SET(bp->b_flags, B_DELWRI); OSAddAtomicLong(1, &nbdwrite); buf_reassign(bp, bp->b_vp); } SET(bp->b_flags, B_DONE); } void buf_markclean(buf_t bp) { if (ISSET(bp->b_flags, B_DELWRI)) { CLR(bp->b_flags, B_DELWRI); OSAddAtomicLong(-1, &nbdwrite); buf_reassign(bp, bp->b_vp); } } void buf_markeintr(buf_t bp) { SET(bp->b_flags, B_EINTR); } void buf_markaged(buf_t bp) { SET(bp->b_flags, B_AGE); } int buf_fua(buf_t bp) { if ((bp->b_flags & B_FUA) == B_FUA) { return 1; } return 0; } void buf_markfua(buf_t bp) { SET(bp->b_flags, B_FUA); } #if CONFIG_PROTECT cpx_t bufattr_cpx(bufattr_t bap) { return bap->ba_cpx; } void bufattr_setcpx(bufattr_t bap, cpx_t cpx) { bap->ba_cpx = cpx; } void buf_setcpoff(buf_t bp, uint64_t foffset) { bp->b_attr.ba_cp_file_off = foffset; } uint64_t bufattr_cpoff(bufattr_t bap) { return bap->ba_cp_file_off; } void bufattr_setcpoff(bufattr_t bap, uint64_t foffset) { bap->ba_cp_file_off = foffset; } #else // !CONTECT_PROTECT uint64_t bufattr_cpoff(bufattr_t bap __unused) { return 0; } void bufattr_setcpoff(__unused bufattr_t bap, __unused uint64_t foffset) { return; } struct cpx * bufattr_cpx(__unused bufattr_t bap) { return NULL; } void bufattr_setcpx(__unused bufattr_t bap, __unused struct cpx *cpx) { } #endif /* !CONFIG_PROTECT */ bufattr_t bufattr_alloc(void) { return kalloc_type(struct bufattr, Z_WAITOK | Z_ZERO); } void bufattr_free(bufattr_t bap) { kfree_type(struct bufattr, bap); } bufattr_t bufattr_dup(bufattr_t bap) { bufattr_t new_bufattr; new_bufattr = kalloc_type(struct bufattr, Z_WAITOK | Z_NOFAIL); /* Copy the provided one into the new copy */ memcpy(new_bufattr, bap, sizeof(struct bufattr)); return new_bufattr; } int bufattr_rawencrypted(bufattr_t bap) { if ((bap->ba_flags & BA_RAW_ENCRYPTED_IO)) { return 1; } return 0; } int bufattr_throttled(bufattr_t bap) { return GET_BUFATTR_IO_TIER(bap); } int bufattr_passive(bufattr_t bap) { if ((bap->ba_flags & BA_PASSIVE)) { return 1; } return 0; } int bufattr_nocache(bufattr_t bap) { if ((bap->ba_flags & BA_NOCACHE)) { return 1; } return 0; } int bufattr_meta(bufattr_t bap) { if ((bap->ba_flags & BA_META)) { return 1; } return 0; } void bufattr_markmeta(bufattr_t bap) { SET(bap->ba_flags, BA_META); } int bufattr_delayidlesleep(bufattr_t bap) { if ((bap->ba_flags & BA_DELAYIDLESLEEP)) { return 1; } return 0; } bufattr_t buf_attr(buf_t bp) { return &bp->b_attr; } void buf_markstatic(buf_t bp __unused) { SET(bp->b_flags, B_STATICCONTENT); } int buf_static(buf_t bp) { if ((bp->b_flags & B_STATICCONTENT)) { return 1; } return 0; } void bufattr_markgreedymode(bufattr_t bap) { SET(bap->ba_flags, BA_GREEDY_MODE); } int bufattr_greedymode(bufattr_t bap) { if ((bap->ba_flags & BA_GREEDY_MODE)) { return 1; } return 0; } void bufattr_markisochronous(bufattr_t bap) { SET(bap->ba_flags, BA_ISOCHRONOUS); } int bufattr_isochronous(bufattr_t bap) { if ((bap->ba_flags & BA_ISOCHRONOUS)) { return 1; } return 0; } void bufattr_markquickcomplete(bufattr_t bap) { SET(bap->ba_flags, BA_QUICK_COMPLETE); } int bufattr_quickcomplete(bufattr_t bap) { if ((bap->ba_flags & BA_QUICK_COMPLETE)) { return 1; } return 0; } void bufattr_markioscheduled(bufattr_t bap) { SET(bap->ba_flags, BA_IO_SCHEDULED); } int bufattr_ioscheduled(bufattr_t bap) { if ((bap->ba_flags & BA_IO_SCHEDULED)) { return 1; } return 0; } void bufattr_markexpeditedmeta(bufattr_t bap) { SET(bap->ba_flags, BA_EXPEDITED_META_IO); } int bufattr_expeditedmeta(bufattr_t bap) { if ((bap->ba_flags & BA_EXPEDITED_META_IO)) { return 1; } return 0; } int bufattr_willverify(bufattr_t bap) { if ((bap->ba_flags & BA_WILL_VERIFY)) { return 1; } return 0; } errno_t buf_error(buf_t bp) { return bp->b_error; } void buf_seterror(buf_t bp, errno_t error) { if ((bp->b_error = error)) { SET(bp->b_flags, B_ERROR); } else { CLR(bp->b_flags, B_ERROR); } } void buf_setflags(buf_t bp, int32_t flags) { SET(bp->b_flags, (flags & BUF_X_WRFLAGS)); } void buf_clearflags(buf_t bp, int32_t flags) { CLR(bp->b_flags, (flags & BUF_X_WRFLAGS)); } int32_t buf_flags(buf_t bp) { return bp->b_flags & BUF_X_RDFLAGS; } void buf_reset(buf_t bp, int32_t io_flags) { CLR(bp->b_flags, (B_READ | B_WRITE | B_ERROR | B_DONE | B_INVAL | B_ASYNC | B_NOCACHE | B_FUA)); SET(bp->b_flags, (io_flags & (B_ASYNC | B_READ | B_WRITE | B_NOCACHE))); bp->b_error = 0; } uint32_t buf_count(buf_t bp) { return bp->b_bcount; } void buf_setcount(buf_t bp, uint32_t bcount) { bp->b_bcount = bcount; } uint32_t buf_size(buf_t bp) { return bp->b_bufsize; } void buf_setsize(buf_t bp, uint32_t bufsize) { bp->b_bufsize = bufsize; } uint32_t buf_resid(buf_t bp) { return bp->b_resid; } void buf_setresid(buf_t bp, uint32_t resid) { bp->b_resid = resid; } uint32_t buf_dirtyoff(buf_t bp) { return bp->b_dirtyoff; } uint32_t buf_dirtyend(buf_t bp) { return bp->b_dirtyend; } void buf_setdirtyoff(buf_t bp, uint32_t dirtyoff) { bp->b_dirtyoff = dirtyoff; } void buf_setdirtyend(buf_t bp, uint32_t dirtyend) { bp->b_dirtyend = dirtyend; } uintptr_t buf_dataptr(buf_t bp) { return bp->b_datap; } void buf_setdataptr(buf_t bp, uintptr_t data) { bp->b_datap = data; } vnode_t buf_vnode(buf_t bp) { return bp->b_vp; } void buf_setvnode(buf_t bp, vnode_t vp) { bp->b_vp = vp; } vnode_t buf_vnop_vnode(buf_t bp) { return bp->b_vnop_vp ? bp->b_vnop_vp : bp->b_vp; } void * buf_callback(buf_t bp) { if (!(bp->b_flags & B_CALL)) { return (void *) NULL; } return (void *)bp->b_iodone; } errno_t buf_setcallback(buf_t bp, void (*callback)(buf_t, void *), void *transaction) { assert(!ISSET(bp->b_flags, B_FILTER) && ISSET(bp->b_lflags, BL_BUSY)); if (callback) { bp->b_flags |= (B_CALL | B_ASYNC); } else { bp->b_flags &= ~B_CALL; } bp->b_transaction = transaction; bp->b_iodone = callback; return 0; } errno_t buf_setupl(buf_t bp, upl_t upl, uint32_t offset) { if (!(bp->b_lflags & BL_IOBUF)) { return EINVAL; } if (upl) { bp->b_flags |= B_CLUSTER; } else { bp->b_flags &= ~B_CLUSTER; } bp->b_upl = upl; bp->b_uploffset = offset; return 0; } buf_t buf_clone(buf_t bp, int io_offset, int io_size, void (*iodone)(buf_t, void *), void *arg) { buf_t io_bp; int add1, add2; if (io_offset < 0 || io_size < 0) { return NULL; } if ((unsigned)(io_offset + io_size) > (unsigned)bp->b_bcount) { return NULL; } if (bp->b_flags & B_CLUSTER) { if (io_offset && ((bp->b_uploffset + io_offset) & PAGE_MASK)) { return NULL; } if (os_add_overflow(io_offset, io_size, &add1) || os_add_overflow(add1, bp->b_uploffset, &add2)) { return NULL; } if ((add2 & PAGE_MASK) && ((uint32_t)add1 < (uint32_t)bp->b_bcount)) { return NULL; } } io_bp = alloc_io_buf(bp->b_vp, 0); io_bp->b_flags = bp->b_flags & (B_COMMIT_UPL | B_META | B_PAGEIO | B_CLUSTER | B_PHYS | B_RAW | B_ASYNC | B_READ | B_FUA); if (iodone) { io_bp->b_transaction = arg; io_bp->b_iodone = iodone; io_bp->b_flags |= B_CALL; } if (bp->b_flags & B_CLUSTER) { io_bp->b_upl = bp->b_upl; io_bp->b_uploffset = bp->b_uploffset + io_offset; } else { io_bp->b_datap = (uintptr_t)(((char *)bp->b_datap) + io_offset); } io_bp->b_bcount = io_size; return io_bp; } int buf_shadow(buf_t bp) { if (bp->b_lflags & BL_SHADOW) { return 1; } return 0; } buf_t buf_create_shadow_priv(buf_t bp, boolean_t force_copy, uintptr_t external_storage, void (*iodone)(buf_t, void *), void *arg) { return buf_create_shadow_internal(bp, force_copy, external_storage, iodone, arg, 1); } buf_t buf_create_shadow(buf_t bp, boolean_t force_copy, uintptr_t external_storage, void (*iodone)(buf_t, void *), void *arg) { return buf_create_shadow_internal(bp, force_copy, external_storage, iodone, arg, 0); } static buf_t buf_create_shadow_internal(buf_t bp, boolean_t force_copy, uintptr_t external_storage, void (*iodone)(buf_t, void *), void *arg, int priv) { buf_t io_bp; KERNEL_DEBUG(0xbbbbc000 | DBG_FUNC_START, bp, 0, 0, 0, 0); if (!(bp->b_flags & B_META) || (bp->b_lflags & BL_IOBUF)) { KERNEL_DEBUG(0xbbbbc000 | DBG_FUNC_END, bp, 0, 0, 0, 0); return NULL; } #ifdef BUF_MAKE_PRIVATE if (bp->b_shadow_ref && bp->b_data_ref == 0 && external_storage == 0) { panic("buf_create_shadow: %p is in the private state (%d, %d)", bp, bp->b_shadow_ref, bp->b_data_ref); } #endif io_bp = alloc_io_buf(bp->b_vp, priv); io_bp->b_flags = bp->b_flags & (B_META | B_ZALLOC | B_ASYNC | B_READ | B_FUA); io_bp->b_blkno = bp->b_blkno; io_bp->b_lblkno = bp->b_lblkno; io_bp->b_lblksize = bp->b_lblksize; if (iodone) { io_bp->b_transaction = arg; io_bp->b_iodone = iodone; io_bp->b_flags |= B_CALL; } if (force_copy == FALSE) { io_bp->b_bcount = bp->b_bcount; io_bp->b_bufsize = bp->b_bufsize; if (external_storage) { io_bp->b_datap = external_storage; #ifdef BUF_MAKE_PRIVATE io_bp->b_data_store = NULL; #endif } else { io_bp->b_datap = bp->b_datap; #ifdef BUF_MAKE_PRIVATE io_bp->b_data_store = bp; #endif } *(buf_t *)(&io_bp->b_orig) = bp; lck_mtx_lock_spin(&buf_mtx); io_bp->b_lflags |= BL_SHADOW; io_bp->b_shadow = bp->b_shadow; bp->b_shadow = io_bp; bp->b_shadow_ref++; #ifdef BUF_MAKE_PRIVATE if (external_storage) { io_bp->b_lflags |= BL_EXTERNAL; } else { bp->b_data_ref++; } #endif lck_mtx_unlock(&buf_mtx); } else { if (external_storage) { #ifdef BUF_MAKE_PRIVATE io_bp->b_lflags |= BL_EXTERNAL; #endif io_bp->b_bcount = bp->b_bcount; io_bp->b_bufsize = bp->b_bufsize; io_bp->b_datap = external_storage; } else { allocbuf(io_bp, bp->b_bcount); io_bp->b_lflags |= BL_IOBUF_ALLOC; } bcopy((caddr_t)bp->b_datap, (caddr_t)io_bp->b_datap, bp->b_bcount); #ifdef BUF_MAKE_PRIVATE io_bp->b_data_store = NULL; #endif } KERNEL_DEBUG(0xbbbbc000 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, io_bp, 0); return io_bp; } #ifdef BUF_MAKE_PRIVATE errno_t buf_make_private(buf_t bp) { buf_t ds_bp; buf_t t_bp; struct buf my_buf; KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_START, bp, bp->b_shadow_ref, 0, 0, 0); if (bp->b_shadow_ref == 0 || bp->b_data_ref == 0 || ISSET(bp->b_lflags, BL_SHADOW)) { KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, EINVAL, 0); return EINVAL; } my_buf.b_flags = B_META; my_buf.b_datap = (uintptr_t)NULL; allocbuf(&my_buf, bp->b_bcount); bcopy((caddr_t)bp->b_datap, (caddr_t)my_buf.b_datap, bp->b_bcount); lck_mtx_lock_spin(&buf_mtx); for (t_bp = bp->b_shadow; t_bp; t_bp = t_bp->b_shadow) { if (!ISSET(bp->b_lflags, BL_EXTERNAL)) { break; } } ds_bp = t_bp; if (ds_bp == NULL && bp->b_data_ref) { panic("buf_make_private: b_data_ref != 0 && ds_bp == NULL"); } if (ds_bp && (bp->b_data_ref == 0 || bp->b_shadow_ref == 0)) { panic("buf_make_private: ref_count == 0 && ds_bp != NULL"); } if (ds_bp == NULL) { lck_mtx_unlock(&buf_mtx); buf_free_meta_store(&my_buf); KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, EINVAL, 0); return EINVAL; } for (t_bp = bp->b_shadow; t_bp; t_bp = t_bp->b_shadow) { if (!ISSET(t_bp->b_lflags, BL_EXTERNAL)) { t_bp->b_data_store = ds_bp; } } ds_bp->b_data_ref = bp->b_data_ref; bp->b_data_ref = 0; bp->b_datap = my_buf.b_datap; lck_mtx_unlock(&buf_mtx); KERNEL_DEBUG(0xbbbbc004 | DBG_FUNC_END, bp, bp->b_shadow_ref, 0, 0, 0); return 0; } #endif void buf_setfilter(buf_t bp, void (*filter)(buf_t, void *), void *transaction, void(**old_iodone)(buf_t, void *), void **old_transaction) { assert(ISSET(bp->b_lflags, BL_BUSY)); if (old_iodone) { *old_iodone = bp->b_iodone; } if (old_transaction) { *old_transaction = bp->b_transaction; } bp->b_transaction = transaction; bp->b_iodone = filter; if (filter) { bp->b_flags |= B_FILTER; } else { bp->b_flags &= ~B_FILTER; } } daddr64_t buf_blkno(buf_t bp) { return bp->b_blkno; } daddr64_t buf_lblkno(buf_t bp) { return bp->b_lblkno; } uint32_t buf_lblksize(buf_t bp) { return bp->b_lblksize; } void buf_setblkno(buf_t bp, daddr64_t blkno) { bp->b_blkno = blkno; } void buf_setlblkno(buf_t bp, daddr64_t lblkno) { bp->b_lblkno = lblkno; } void buf_setlblksize(buf_t bp, uint32_t lblksize) { bp->b_lblksize = lblksize; } dev_t buf_device(buf_t bp) { return bp->b_dev; } errno_t buf_setdevice(buf_t bp, vnode_t vp) { if ((vp->v_type != VBLK) && (vp->v_type != VCHR)) { return EINVAL; } bp->b_dev = vp->v_rdev; return 0; } void * buf_drvdata(buf_t bp) { return bp->b_drvdata; } void buf_setdrvdata(buf_t bp, void *drvdata) { bp->b_drvdata = drvdata; } void * buf_fsprivate(buf_t bp) { return bp->b_fsprivate; } void buf_setfsprivate(buf_t bp, void *fsprivate) { bp->b_fsprivate = fsprivate; } kauth_cred_t buf_rcred(buf_t bp) { return bp->b_rcred; } kauth_cred_t buf_wcred(buf_t bp) { return bp->b_wcred; } void * buf_upl(buf_t bp) { return bp->b_upl; } uint32_t buf_uploffset(buf_t bp) { return (uint32_t)(bp->b_uploffset); } proc_t buf_proc(buf_t bp) { return bp->b_proc; } static errno_t buf_map_range_internal(buf_t bp, caddr_t *io_addr, boolean_t legacymode, vm_prot_t prot) { buf_t real_bp; vm_offset_t vaddr; kern_return_t kret; if (!(bp->b_flags & B_CLUSTER)) { *io_addr = (caddr_t)bp->b_datap; return 0; } real_bp = (buf_t)(bp->b_real_bp); if (real_bp && real_bp->b_datap) { /* * b_real_bp is only valid if B_CLUSTER is SET * if it's non-zero, than someone did a cluster_bp call * if the backing physical pages were already mapped * in before the call to cluster_bp (non-zero b_datap), * than we just use that mapping */ *io_addr = (caddr_t)real_bp->b_datap; return 0; } if (legacymode) { kret = ubc_upl_map(bp->b_upl, &vaddr); /* Map it in */ if (kret == KERN_SUCCESS) { vaddr += bp->b_uploffset; } } else { kret = ubc_upl_map_range(bp->b_upl, bp->b_uploffset, bp->b_bcount, prot, &vaddr); /* Map it in */ } if (kret != KERN_SUCCESS) { *io_addr = NULL; return ENOMEM; } *io_addr = (caddr_t)vaddr; return 0; } errno_t buf_map_range(buf_t bp, caddr_t *io_addr) { return buf_map_range_internal(bp, io_addr, false, VM_PROT_DEFAULT); } errno_t buf_map_range_with_prot(buf_t bp, caddr_t *io_addr, vm_prot_t prot) { /* Only VM_PROT_READ and/or VM_PROT_WRITE is allowed. */ prot &= (VM_PROT_READ | VM_PROT_WRITE); if (prot == VM_PROT_NONE) { *io_addr = NULL; return EINVAL; } return buf_map_range_internal(bp, io_addr, false, prot); } errno_t buf_map(buf_t bp, caddr_t *io_addr) { return buf_map_range_internal(bp, io_addr, true, VM_PROT_DEFAULT); } static errno_t buf_unmap_range_internal(buf_t bp, boolean_t legacymode) { buf_t real_bp; kern_return_t kret; if (!(bp->b_flags & B_CLUSTER)) { return 0; } /* * see buf_map for the explanation */ real_bp = (buf_t)(bp->b_real_bp); if (real_bp && real_bp->b_datap) { return 0; } if ((bp->b_lflags & BL_IOBUF) && ((bp->b_flags & (B_PAGEIO | B_READ)) != (B_PAGEIO | B_READ))) { /* * ignore pageins... the 'right' thing will * happen due to the way we handle speculative * clusters... * * when we commit these pages, we'll hit * it with UPL_COMMIT_INACTIVE which * will clear the reference bit that got * turned on when we touched the mapping */ bp->b_flags |= B_AGE; } if (legacymode) { kret = ubc_upl_unmap(bp->b_upl); } else { kret = ubc_upl_unmap_range(bp->b_upl, bp->b_uploffset, bp->b_bcount); } if (kret != KERN_SUCCESS) { return EINVAL; } return 0; } errno_t buf_unmap_range(buf_t bp) { return buf_unmap_range_internal(bp, false); } errno_t buf_unmap(buf_t bp) { return buf_unmap_range_internal(bp, true); } void buf_clear(buf_t bp) { caddr_t baddr; if (buf_map(bp, &baddr) == 0) { bzero(baddr, bp->b_bcount); buf_unmap(bp); } bp->b_resid = 0; } /* * Read or write a buffer that is not contiguous on disk. * buffer is marked done/error at the conclusion */ static int buf_strategy_fragmented(vnode_t devvp, buf_t bp, off_t f_offset, size_t contig_bytes) { vnode_t vp = buf_vnode(bp); buf_t io_bp; /* For reading or writing a single block */ int io_direction; int io_resid; size_t io_contig_bytes; daddr64_t io_blkno; int error = 0; int bmap_flags; /* * save our starting point... the bp was already mapped * in buf_strategy before we got called * no sense doing it again. */ io_blkno = bp->b_blkno; /* * Make sure we redo this mapping for the next I/O * i.e. this can never be a 'permanent' mapping */ bp->b_blkno = bp->b_lblkno; /* * Get an io buffer to do the deblocking */ io_bp = alloc_io_buf(devvp, 0); io_bp->b_lblkno = bp->b_lblkno; io_bp->b_lblksize = bp->b_lblksize; io_bp->b_datap = bp->b_datap; io_resid = bp->b_bcount; io_direction = bp->b_flags & B_READ; io_contig_bytes = contig_bytes; if (bp->b_flags & B_READ) { bmap_flags = VNODE_READ; } else { bmap_flags = VNODE_WRITE; } for (;;) { if (io_blkno == -1) { /* * this is unexepected, but we'll allow for it */ bzero((caddr_t)io_bp->b_datap, (int)io_contig_bytes); } else { io_bp->b_bcount = (uint32_t)io_contig_bytes; io_bp->b_bufsize = (uint32_t)io_contig_bytes; io_bp->b_resid = (uint32_t)io_contig_bytes; io_bp->b_blkno = io_blkno; buf_reset(io_bp, io_direction); /* * Call the device to do the I/O and wait for it. Make sure the appropriate party is charged for write */ if (!ISSET(bp->b_flags, B_READ)) { OSAddAtomic(1, &devvp->v_numoutput); } if ((error = VNOP_STRATEGY(io_bp))) { break; } if ((error = (int)buf_biowait(io_bp))) { break; } if (io_bp->b_resid) { io_resid -= (io_contig_bytes - io_bp->b_resid); break; } } if ((io_resid -= io_contig_bytes) == 0) { break; } f_offset += io_contig_bytes; io_bp->b_datap += io_contig_bytes; /* * Map the current position to a physical block number */ if ((error = VNOP_BLOCKMAP(vp, f_offset, io_resid, &io_blkno, &io_contig_bytes, NULL, bmap_flags, NULL))) { break; } } buf_free(io_bp); if (error) { buf_seterror(bp, error); } bp->b_resid = io_resid; /* * This I/O is now complete */ buf_biodone(bp); return error; } /* * struct vnop_strategy_args { * struct buf *a_bp; * } *ap; */ errno_t buf_strategy(vnode_t devvp, void *ap) { buf_t bp = ((struct vnop_strategy_args *)ap)->a_bp; vnode_t vp = bp->b_vp; int bmap_flags; errno_t error; #if CONFIG_DTRACE int dtrace_io_start_flag = 0; /* We only want to trip the io:::start * probe once, with the true physical * block in place (b_blkno) */ #endif if (vp == NULL || vp->v_type == VCHR || vp->v_type == VBLK) { panic("buf_strategy: b_vp == NULL || vtype == VCHR | VBLK"); } /* * associate the physical device with * with this buf_t even if we don't * end up issuing the I/O... */ bp->b_dev = devvp->v_rdev; if (bp->b_flags & B_READ) { bmap_flags = VNODE_READ; } else { bmap_flags = VNODE_WRITE; } if (!(bp->b_flags & B_CLUSTER)) { if ((bp->b_upl)) { /* * we have a UPL associated with this bp * go through cluster_bp which knows how * to deal with filesystem block sizes * that aren't equal to the page size */ DTRACE_IO1(start, buf_t, bp); return cluster_bp(bp); } if (bp->b_blkno == bp->b_lblkno) { off_t f_offset; size_t contig_bytes; if (bp->b_lblksize && bp->b_lblkno >= 0) { f_offset = bp->b_lblkno * bp->b_lblksize; } else if ((error = VNOP_BLKTOOFF(vp, bp->b_lblkno, &f_offset))) { DTRACE_IO1(start, buf_t, bp); buf_seterror(bp, error); buf_biodone(bp); return error; } if ((error = VNOP_BLOCKMAP(vp, f_offset, bp->b_bcount, &bp->b_blkno, &contig_bytes, NULL, bmap_flags, NULL))) { DTRACE_IO1(start, buf_t, bp); buf_seterror(bp, error); buf_biodone(bp); return error; } DTRACE_IO1(start, buf_t, bp); #if CONFIG_DTRACE dtrace_io_start_flag = 1; #endif /* CONFIG_DTRACE */ if ((bp->b_blkno == -1) || (contig_bytes == 0)) { /* Set block number to force biodone later */ bp->b_blkno = -1; buf_clear(bp); } else if (contig_bytes < (size_t)bp->b_bcount) { return buf_strategy_fragmented(devvp, bp, f_offset, contig_bytes); } } #if CONFIG_DTRACE if (dtrace_io_start_flag == 0) { DTRACE_IO1(start, buf_t, bp); dtrace_io_start_flag = 1; } #endif /* CONFIG_DTRACE */ if (bp->b_blkno == -1) { buf_biodone(bp); return 0; } } #if CONFIG_DTRACE if (dtrace_io_start_flag == 0) { DTRACE_IO1(start, buf_t, bp); } #endif /* CONFIG_DTRACE */ #if CONFIG_PROTECT /* Capture f_offset in the bufattr*/ cpx_t cpx = bufattr_cpx(buf_attr(bp)); if (cpx) { /* No need to go here for older EAs */ if (cpx_use_offset_for_iv(cpx) && !cpx_synthetic_offset_for_iv(cpx)) { off_t f_offset; /* * this assert should be changed if cluster_io ever * changes its logical block size. */ assert((bp->b_lblksize == CLUSTER_IO_BLOCK_SIZE) || !(bp->b_flags & B_CLUSTER)); if (bp->b_lblksize && bp->b_lblkno >= 0) { f_offset = bp->b_lblkno * bp->b_lblksize; } else if ((error = VNOP_BLKTOOFF(bp->b_vp, bp->b_lblkno, &f_offset))) { return error; } /* * Attach the file offset to this buffer. The * bufattr attributes will be passed down the stack * until they reach the storage driver (whether * IOFlashStorage, ASP, or IONVMe). The driver * will retain the offset in a local variable when it * issues its I/Os to the NAND controller. * * Note that LwVM may end up splitting this I/O * into sub-I/Os if it crosses a chunk boundary. In this * case, LwVM will update this field when it dispatches * each I/O to IOFlashStorage. But from our perspective * we have only issued a single I/O. * * In the case of APFS we do not bounce through another * intermediate layer (such as CoreStorage). APFS will * issue the I/Os directly to the block device / IOMedia * via buf_strategy on the specfs node. */ buf_setcpoff(bp, f_offset); CP_DEBUG((CPDBG_OFFSET_IO | DBG_FUNC_NONE), (uint32_t) f_offset, (uint32_t) bp->b_lblkno, (uint32_t) bp->b_blkno, (uint32_t) bp->b_bcount, 0); } } #endif /* * we can issue the I/O because... * either B_CLUSTER is set which * means that the I/O is properly set * up to be a multiple of the page size, or * we were able to successfully set up the * physical block mapping */ bp->b_vnop_vp = devvp; error = VOCALL(devvp->v_op, VOFFSET(vnop_strategy), ap); bp->b_vnop_vp = NULLVP; DTRACE_FSINFO(strategy, vnode_t, vp); return error; } buf_t buf_alloc(vnode_t vp) { return alloc_io_buf(vp, is_vm_privileged()); } void buf_free(buf_t bp) { free_io_buf(bp); } /* * iterate buffers for the specified vp. * if BUF_SCAN_DIRTY is set, do the dirty list * if BUF_SCAN_CLEAN is set, do the clean list * if neither flag is set, default to BUF_SCAN_DIRTY * if BUF_NOTIFY_BUSY is set, call the callout function using a NULL bp for busy pages */ struct buf_iterate_info_t { int flag; struct buflists *listhead; }; void buf_iterate(vnode_t vp, int (*callout)(buf_t, void *), int flags, void *arg) { buf_t bp; int retval; struct buflists local_iterblkhd; int lock_flags = BAC_NOWAIT | BAC_REMOVE; int notify_busy = flags & BUF_NOTIFY_BUSY; struct buf_iterate_info_t list[2]; int num_lists, i; if (flags & BUF_SKIP_LOCKED) { lock_flags |= BAC_SKIP_LOCKED; } if (flags & BUF_SKIP_NONLOCKED) { lock_flags |= BAC_SKIP_NONLOCKED; } if (!(flags & (BUF_SCAN_DIRTY | BUF_SCAN_CLEAN))) { flags |= BUF_SCAN_DIRTY; } num_lists = 0; if (flags & BUF_SCAN_DIRTY) { list[num_lists].flag = VBI_DIRTY; list[num_lists].listhead = &vp->v_dirtyblkhd; num_lists++; } if (flags & BUF_SCAN_CLEAN) { list[num_lists].flag = VBI_CLEAN; list[num_lists].listhead = &vp->v_cleanblkhd; num_lists++; } for (i = 0; i < num_lists; i++) { lck_mtx_lock(&buf_mtx); if (buf_iterprepare(vp, &local_iterblkhd, list[i].flag)) { lck_mtx_unlock(&buf_mtx); continue; } while (!LIST_EMPTY(&local_iterblkhd)) { bp = LIST_FIRST(&local_iterblkhd); LIST_REMOVE(bp, b_vnbufs); LIST_INSERT_HEAD(list[i].listhead, bp, b_vnbufs); if (buf_acquire_locked(bp, lock_flags, 0, 0)) { if (notify_busy) { bp = NULL; } else { continue; } } lck_mtx_unlock(&buf_mtx); retval = callout(bp, arg); switch (retval) { case BUF_RETURNED: if (bp) { buf_brelse(bp); } break; case BUF_CLAIMED: break; case BUF_RETURNED_DONE: if (bp) { buf_brelse(bp); } lck_mtx_lock(&buf_mtx); goto out; case BUF_CLAIMED_DONE: lck_mtx_lock(&buf_mtx); goto out; } lck_mtx_lock(&buf_mtx); } /* while list has more nodes */ out: buf_itercomplete(vp, &local_iterblkhd, list[i].flag); lck_mtx_unlock(&buf_mtx); } /* for each list */ } /* buf_iterate */ /* * Flush out and invalidate all buffers associated with a vnode. */ int buf_invalidateblks(vnode_t vp, int flags, int slpflag, int slptimeo) { buf_t bp; int aflags; int error = 0; int must_rescan = 1; struct buflists local_iterblkhd; if (LIST_EMPTY(&vp->v_cleanblkhd) && LIST_EMPTY(&vp->v_dirtyblkhd)) { return 0; } lck_mtx_lock(&buf_mtx); for (;;) { if (must_rescan == 0) { /* * the lists may not be empty, but all that's left at this * point are metadata or B_LOCKED buffers which are being * skipped... we know this because we made it through both * the clean and dirty lists without dropping buf_mtx... * each time we drop buf_mtx we bump "must_rescan" */ break; } if (LIST_EMPTY(&vp->v_cleanblkhd) && LIST_EMPTY(&vp->v_dirtyblkhd)) { break; } must_rescan = 0; /* * iterate the clean list */ if (buf_iterprepare(vp, &local_iterblkhd, VBI_CLEAN)) { goto try_dirty_list; } while (!LIST_EMPTY(&local_iterblkhd)) { bp = LIST_FIRST(&local_iterblkhd); LIST_REMOVE(bp, b_vnbufs); LIST_INSERT_HEAD(&vp->v_cleanblkhd, bp, b_vnbufs); /* * some filesystems distinguish meta data blocks with a negative logical block # */ if ((flags & BUF_SKIP_META) && (bp->b_lblkno < 0 || ISSET(bp->b_flags, B_META))) { continue; } aflags = BAC_REMOVE; if (!(flags & BUF_INVALIDATE_LOCKED)) { aflags |= BAC_SKIP_LOCKED; } if ((error = (int)buf_acquire_locked(bp, aflags, slpflag, slptimeo))) { if (error == EDEADLK) { /* * this buffer was marked B_LOCKED... * we didn't drop buf_mtx, so we * we don't need to rescan */ continue; } if (error == EAGAIN) { /* * found a busy buffer... we blocked and * dropped buf_mtx, so we're going to * need to rescan after this pass is completed */ must_rescan++; continue; } /* * got some kind of 'real' error out of the msleep * in buf_acquire_locked, terminate the scan and return the error */ buf_itercomplete(vp, &local_iterblkhd, VBI_CLEAN); lck_mtx_unlock(&buf_mtx); return error; } lck_mtx_unlock(&buf_mtx); if (bp->b_flags & B_LOCKED) { KERNEL_DEBUG(0xbbbbc038, bp, 0, 0, 0, 0); } CLR(bp->b_flags, B_LOCKED); SET(bp->b_flags, B_INVAL); buf_brelse(bp); lck_mtx_lock(&buf_mtx); /* * by dropping buf_mtx, we allow new * buffers to be added to the vnode list(s) * we'll have to rescan at least once more * if the queues aren't empty */ must_rescan++; } buf_itercomplete(vp, &local_iterblkhd, VBI_CLEAN); try_dirty_list: /* * Now iterate on dirty blks */ if (buf_iterprepare(vp, &local_iterblkhd, VBI_DIRTY)) { continue; } while (!LIST_EMPTY(&local_iterblkhd)) { bp = LIST_FIRST(&local_iterblkhd); LIST_REMOVE(bp, b_vnbufs); LIST_INSERT_HEAD(&vp->v_dirtyblkhd, bp, b_vnbufs); /* * some filesystems distinguish meta data blocks with a negative logical block # */ if ((flags & BUF_SKIP_META) && (bp->b_lblkno < 0 || ISSET(bp->b_flags, B_META))) { continue; } aflags = BAC_REMOVE; if (!(flags & BUF_INVALIDATE_LOCKED)) { aflags |= BAC_SKIP_LOCKED; } if ((error = (int)buf_acquire_locked(bp, aflags, slpflag, slptimeo))) { if (error == EDEADLK) { /* * this buffer was marked B_LOCKED... * we didn't drop buf_mtx, so we * we don't need to rescan */ continue; } if (error == EAGAIN) { /* * found a busy buffer... we blocked and * dropped buf_mtx, so we're going to * need to rescan after this pass is completed */ must_rescan++; continue; } /* * got some kind of 'real' error out of the msleep * in buf_acquire_locked, terminate the scan and return the error */ buf_itercomplete(vp, &local_iterblkhd, VBI_DIRTY); lck_mtx_unlock(&buf_mtx); return error; } lck_mtx_unlock(&buf_mtx); if (bp->b_flags & B_LOCKED) { KERNEL_DEBUG(0xbbbbc038, bp, 0, 0, 1, 0); } CLR(bp->b_flags, B_LOCKED); SET(bp->b_flags, B_INVAL); if (ISSET(bp->b_flags, B_DELWRI) && (flags & BUF_WRITE_DATA)) { (void) VNOP_BWRITE(bp); } else { buf_brelse(bp); } lck_mtx_lock(&buf_mtx); /* * by dropping buf_mtx, we allow new * buffers to be added to the vnode list(s) * we'll have to rescan at least once more * if the queues aren't empty */ must_rescan++; } buf_itercomplete(vp, &local_iterblkhd, VBI_DIRTY); } lck_mtx_unlock(&buf_mtx); return 0; } void buf_flushdirtyblks(vnode_t vp, int wait, int flags, const char *msg) { (void) buf_flushdirtyblks_skipinfo(vp, wait, flags, msg); return; } int buf_flushdirtyblks_skipinfo(vnode_t vp, int wait, int flags, const char *msg) { buf_t bp; int writes_issued = 0; errno_t error; int busy = 0; struct buflists local_iterblkhd; int lock_flags = BAC_NOWAIT | BAC_REMOVE; int any_locked = 0; if (flags & BUF_SKIP_LOCKED) { lock_flags |= BAC_SKIP_LOCKED; } if (flags & BUF_SKIP_NONLOCKED) { lock_flags |= BAC_SKIP_NONLOCKED; } loop: lck_mtx_lock(&buf_mtx); if (buf_iterprepare(vp, &local_iterblkhd, VBI_DIRTY) == 0) { while (!LIST_EMPTY(&local_iterblkhd)) { bp = LIST_FIRST(&local_iterblkhd); LIST_REMOVE(bp, b_vnbufs); LIST_INSERT_HEAD(&vp->v_dirtyblkhd, bp, b_vnbufs); if ((error = buf_acquire_locked(bp, lock_flags, 0, 0)) == EBUSY) { busy++; } if (error) { /* * If we passed in BUF_SKIP_LOCKED or BUF_SKIP_NONLOCKED, * we may want to do somethign differently if a locked or unlocked * buffer was encountered (depending on the arg specified). * In this case, we know that one of those two was set, and the * buf acquisition failed above. * * If it failed with EDEADLK, then save state which can be emitted * later on to the caller. Most callers should not care. */ if (error == EDEADLK) { any_locked++; } continue; } lck_mtx_unlock(&buf_mtx); bp->b_flags &= ~B_LOCKED; /* * Wait for I/O associated with indirect blocks to complete, * since there is no way to quickly wait for them below. */ if ((bp->b_vp == vp) || (wait == 0)) { (void) buf_bawrite(bp); } else { (void) VNOP_BWRITE(bp); } writes_issued++; lck_mtx_lock(&buf_mtx); } buf_itercomplete(vp, &local_iterblkhd, VBI_DIRTY); } lck_mtx_unlock(&buf_mtx); if (wait) { (void)vnode_waitforwrites(vp, 0, 0, 0, msg); if (vp->v_dirtyblkhd.lh_first && busy) { /* * we had one or more BUSY buffers on * the dirtyblock list... most likely * these are due to delayed writes that * were moved to the bclean queue but * have not yet been 'written'. * if we issued some writes on the * previous pass, we try again immediately * if we didn't, we'll sleep for some time * to allow the state to change... */ if (writes_issued == 0) { (void)tsleep((caddr_t)&vp->v_numoutput, PRIBIO + 1, "vnode_flushdirtyblks", hz / 20); } writes_issued = 0; busy = 0; goto loop; } } return any_locked; } /* * called with buf_mtx held... * this lock protects the queue manipulation */ static int buf_iterprepare(vnode_t vp, struct buflists *iterheadp, int flags) { struct buflists * listheadp; if (flags & VBI_DIRTY) { listheadp = &vp->v_dirtyblkhd; } else { listheadp = &vp->v_cleanblkhd; } while (vp->v_iterblkflags & VBI_ITER) { vp->v_iterblkflags |= VBI_ITERWANT; msleep(&vp->v_iterblkflags, &buf_mtx, 0, "buf_iterprepare", NULL); } if (LIST_EMPTY(listheadp)) { LIST_INIT(iterheadp); return EINVAL; } vp->v_iterblkflags |= VBI_ITER; iterheadp->lh_first = listheadp->lh_first; listheadp->lh_first->b_vnbufs.le_prev = &iterheadp->lh_first; LIST_INIT(listheadp); return 0; } /* * called with buf_mtx held... * this lock protects the queue manipulation */ static void buf_itercomplete(vnode_t vp, struct buflists *iterheadp, int flags) { struct buflists * listheadp; buf_t bp; if (flags & VBI_DIRTY) { listheadp = &vp->v_dirtyblkhd; } else { listheadp = &vp->v_cleanblkhd; } while (!LIST_EMPTY(iterheadp)) { bp = LIST_FIRST(iterheadp); LIST_REMOVE(bp, b_vnbufs); LIST_INSERT_HEAD(listheadp, bp, b_vnbufs); } vp->v_iterblkflags &= ~VBI_ITER; if (vp->v_iterblkflags & VBI_ITERWANT) { vp->v_iterblkflags &= ~VBI_ITERWANT; wakeup(&vp->v_iterblkflags); } } static void bremfree_locked(buf_t bp) { struct bqueues *dp = NULL; int whichq; whichq = bp->b_whichq; if (whichq == -1) { if (bp->b_shadow_ref == 0) { panic("bremfree_locked: %p not on freelist", bp); } /* * there are clones pointing to 'bp'... * therefore, it was not put on a freelist * when buf_brelse was last called on 'bp' */ return; } /* * We only calculate the head of the freelist when removing * the last element of the list as that is the only time that * it is needed (e.g. to reset the tail pointer). * * NB: This makes an assumption about how tailq's are implemented. */ if (bp->b_freelist.tqe_next == NULL) { dp = &bufqueues[whichq]; if (dp->tqh_last != &bp->b_freelist.tqe_next) { panic("bremfree: lost tail"); } } TAILQ_REMOVE(dp, bp, b_freelist); if (whichq == BQ_LAUNDRY) { blaundrycnt--; } bp->b_whichq = -1; bp->b_timestamp = 0; bp->b_shadow = 0; } /* * Associate a buffer with a vnode. * buf_mtx must be locked on entry */ static void bgetvp_locked(vnode_t vp, buf_t bp) { if (bp->b_vp != vp) { panic("bgetvp_locked: not free"); } if (vp->v_type == VBLK || vp->v_type == VCHR) { bp->b_dev = vp->v_rdev; } else { bp->b_dev = NODEV; } /* * Insert onto list for new vnode. */ bufinsvn(bp, &vp->v_cleanblkhd); } /* * Disassociate a buffer from a vnode. * buf_mtx must be locked on entry */ static void brelvp_locked(buf_t bp) { /* * Delete from old vnode list, if on one. */ if (bp->b_vnbufs.le_next != NOLIST) { bufremvn(bp); } bp->b_vp = (vnode_t)NULL; } /* * Reassign a buffer from one vnode to another. * Used to assign file specific control information * (indirect blocks) to the vnode to which they belong. */ static void buf_reassign(buf_t bp, vnode_t newvp) { struct buflists *listheadp; if (newvp == NULL) { printf("buf_reassign: NULL"); return; } lck_mtx_lock_spin(&buf_mtx); /* * Delete from old vnode list, if on one. */ if (bp->b_vnbufs.le_next != NOLIST) { bufremvn(bp); } /* * If dirty, put on list of dirty buffers; * otherwise insert onto list of clean buffers. */ if (ISSET(bp->b_flags, B_DELWRI)) { listheadp = &newvp->v_dirtyblkhd; } else { listheadp = &newvp->v_cleanblkhd; } bufinsvn(bp, listheadp); lck_mtx_unlock(&buf_mtx); } static __inline__ void bufhdrinit(buf_t bp) { bzero((char *)bp, sizeof *bp); bp->b_dev = NODEV; bp->b_rcred = NOCRED; bp->b_wcred = NOCRED; bp->b_vnbufs.le_next = NOLIST; bp->b_flags = B_INVAL; return; } /* * Initialize buffers and hash links for buffers. */ __private_extern__ void bufinit(void) { buf_t bp; struct bqueues *dp; int i; nbuf_headers = 0; /* Initialize the buffer queues ('freelists') and the hash table */ for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) { TAILQ_INIT(dp); } bufhashtbl = hashinit(nbuf_hashelements, M_CACHE, &bufhash); buf_busycount = 0; /* Initialize the buffer headers */ for (i = 0; i < max_nbuf_headers; i++) { nbuf_headers++; bp = &buf_headers[i]; bufhdrinit(bp); BLISTNONE(bp); dp = &bufqueues[BQ_EMPTY]; bp->b_whichq = BQ_EMPTY; bp->b_timestamp = buf_timestamp(); binsheadfree(bp, dp, BQ_EMPTY); binshash(bp, &invalhash); } boot_nbuf_headers = nbuf_headers; TAILQ_INIT(&iobufqueue); TAILQ_INIT(&delaybufqueue); for (; i < nbuf_headers + niobuf_headers; i++) { bp = &buf_headers[i]; bufhdrinit(bp); bp->b_whichq = -1; binsheadfree(bp, &iobufqueue, -1); } /* * allocate and initialize cluster specific global locks... */ cluster_init(); printf("using %d buffer headers and %d cluster IO buffer headers\n", nbuf_headers, niobuf_headers); /* start the bcleanbuf() thread */ bcleanbuf_thread_init(); /* Register a callout for relieving vm pressure */ if (vm_set_buffer_cleanup_callout(buffer_cache_gc) != KERN_SUCCESS) { panic("Couldn't register buffer cache callout for vm pressure!"); } } /* * Zones for the meta data buffers */ #define MINMETA 512 #define MAXMETA 16384 KALLOC_HEAP_DEFINE(KHEAP_VFS_BIO, "vfs_bio", KHEAP_ID_DATA_BUFFERS); static struct buf * bio_doread(vnode_t vp, daddr64_t blkno, int size, kauth_cred_t cred, int async, int queuetype) { buf_t bp; bp = buf_getblk(vp, blkno, size, 0, 0, queuetype); /* * If buffer does not have data valid, start a read. * Note that if buffer is B_INVAL, buf_getblk() won't return it. * Therefore, it's valid if it's I/O has completed or been delayed. */ if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { struct proc *p; p = current_proc(); /* Start I/O for the buffer (keeping credentials). */ SET(bp->b_flags, B_READ | async); if (IS_VALID_CRED(cred) && !IS_VALID_CRED(bp->b_rcred)) { kauth_cred_ref(cred); bp->b_rcred = cred; } VNOP_STRATEGY(bp); trace(TR_BREADMISS, pack(vp, size), blkno); /* Pay for the read. */ if (p && p->p_stats) { OSIncrementAtomicLong(&p->p_stats->p_ru.ru_inblock); /* XXX */ } if (async) { /* * since we asked for an ASYNC I/O * the biodone will do the brelse * we don't want to pass back a bp * that we don't 'own' */ bp = NULL; } } else if (async) { buf_brelse(bp); bp = NULL; } trace(TR_BREADHIT, pack(vp, size), blkno); return bp; } /* * Perform the reads for buf_breadn() and buf_meta_breadn(). * Trivial modification to the breada algorithm presented in Bach (p.55). */ static errno_t do_breadn_for_type(vnode_t vp, daddr64_t blkno, int size, daddr64_t *rablks, int *rasizes, int nrablks, kauth_cred_t cred, buf_t *bpp, int queuetype) { buf_t bp; int i; bp = *bpp = bio_doread(vp, blkno, size, cred, 0, queuetype); /* * For each of the read-ahead blocks, start a read, if necessary. */ for (i = 0; i < nrablks; i++) { /* If it's in the cache, just go on to next one. */ if (incore(vp, rablks[i])) { continue; } /* Get a buffer for the read-ahead block */ (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC, queuetype); } /* Otherwise, we had to start a read for it; wait until it's valid. */ return buf_biowait(bp); } /* * Read a disk block. * This algorithm described in Bach (p.54). */ errno_t buf_bread(vnode_t vp, daddr64_t blkno, int size, kauth_cred_t cred, buf_t *bpp) { buf_t bp; /* Get buffer for block. */ bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_READ); /* Wait for the read to complete, and return result. */ return buf_biowait(bp); } /* * Read a disk block. [bread() for meta-data] * This algorithm described in Bach (p.54). */ errno_t buf_meta_bread(vnode_t vp, daddr64_t blkno, int size, kauth_cred_t cred, buf_t *bpp) { buf_t bp; /* Get buffer for block. */ bp = *bpp = bio_doread(vp, blkno, size, cred, 0, BLK_META); /* Wait for the read to complete, and return result. */ return buf_biowait(bp); } /* * Read-ahead multiple disk blocks. The first is sync, the rest async. */ errno_t buf_breadn(vnode_t vp, daddr64_t blkno, int size, daddr64_t *rablks, int *rasizes, int nrablks, kauth_cred_t cred, buf_t *bpp) { return do_breadn_for_type(vp, blkno, size, rablks, rasizes, nrablks, cred, bpp, BLK_READ); } /* * Read-ahead multiple disk blocks. The first is sync, the rest async. * [buf_breadn() for meta-data] */ errno_t buf_meta_breadn(vnode_t vp, daddr64_t blkno, int size, daddr64_t *rablks, int *rasizes, int nrablks, kauth_cred_t cred, buf_t *bpp) { return do_breadn_for_type(vp, blkno, size, rablks, rasizes, nrablks, cred, bpp, BLK_META); } /* * Block write. Described in Bach (p.56) */ errno_t buf_bwrite(buf_t bp) { int sync, wasdelayed; errno_t rv; proc_t p = current_proc(); vnode_t vp = bp->b_vp; if (bp->b_datap == 0) { if (brecover_data(bp) == 0) { return 0; } } /* Remember buffer type, to switch on it later. */ sync = !ISSET(bp->b_flags, B_ASYNC); wasdelayed = ISSET(bp->b_flags, B_DELWRI); CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); if (wasdelayed) { OSAddAtomicLong(-1, &nbdwrite); } if (!sync) { /* * If not synchronous, pay for the I/O operation and make * sure the buf is on the correct vnode queue. We have * to do this now, because if we don't, the vnode may not * be properly notified that its I/O has completed. */ if (wasdelayed) { buf_reassign(bp, vp); } else if (p && p->p_stats) { OSIncrementAtomicLong(&p->p_stats->p_ru.ru_oublock); /* XXX */ } } trace(TR_BUFWRITE, pack(vp, bp->b_bcount), bp->b_lblkno); /* Initiate disk write. Make sure the appropriate party is charged. */ OSAddAtomic(1, &vp->v_numoutput); VNOP_STRATEGY(bp); if (sync) { /* * If I/O was synchronous, wait for it to complete. */ rv = buf_biowait(bp); /* * Pay for the I/O operation, if it's not been paid for, and * make sure it's on the correct vnode queue. (async operatings * were payed for above.) */ if (wasdelayed) { buf_reassign(bp, vp); } else if (p && p->p_stats) { OSIncrementAtomicLong(&p->p_stats->p_ru.ru_oublock); /* XXX */ } /* Release the buffer. */ buf_brelse(bp); return rv; } else { return 0; } } int vn_bwrite(struct vnop_bwrite_args *ap) { return buf_bwrite(ap->a_bp); } /* * Delayed write. * * The buffer is marked dirty, but is not queued for I/O. * This routine should be used when the buffer is expected * to be modified again soon, typically a small write that * partially fills a buffer. * * NB: magnetic tapes cannot be delayed; they must be * written in the order that the writes are requested. * * Described in Leffler, et al. (pp. 208-213). * * Note: With the ability to allocate additional buffer * headers, we can get in to the situation where "too" many * buf_bdwrite()s can create situation where the kernel can create * buffers faster than the disks can service. Doing a buf_bawrite() in * cases where we have "too many" outstanding buf_bdwrite()s avoids that. */ int bdwrite_internal(buf_t bp, int return_error) { proc_t p = current_proc(); vnode_t vp = bp->b_vp; /* * If the block hasn't been seen before: * (1) Mark it as having been seen, * (2) Charge for the write. * (3) Make sure it's on its vnode's correct block list, */ if (!ISSET(bp->b_flags, B_DELWRI)) { SET(bp->b_flags, B_DELWRI); if (p && p->p_stats) { OSIncrementAtomicLong(&p->p_stats->p_ru.ru_oublock); /* XXX */ } OSAddAtomicLong(1, &nbdwrite); buf_reassign(bp, vp); } /* * if we're not LOCKED, but the total number of delayed writes * has climbed above 75% of the total buffers in the system * return an error if the caller has indicated that it can * handle one in this case, otherwise schedule the I/O now * this is done to prevent us from allocating tons of extra * buffers when dealing with virtual disks (i.e. DiskImages), * because additional buffers are dynamically allocated to prevent * deadlocks from occurring * * however, can't do a buf_bawrite() if the LOCKED bit is set because the * buffer is part of a transaction and can't go to disk until * the LOCKED bit is cleared. */ if (!ISSET(bp->b_flags, B_LOCKED) && nbdwrite > ((nbuf_headers / 4) * 3)) { if (return_error) { return EAGAIN; } /* * If the vnode has "too many" write operations in progress * wait for them to finish the IO */ (void)vnode_waitforwrites(vp, VNODE_ASYNC_THROTTLE, 0, 0, "buf_bdwrite"); return buf_bawrite(bp); } /* Otherwise, the "write" is done, so mark and release the buffer. */ SET(bp->b_flags, B_DONE); buf_brelse(bp); return 0; } errno_t buf_bdwrite(buf_t bp) { return bdwrite_internal(bp, 0); } /* * Asynchronous block write; just an asynchronous buf_bwrite(). * * Note: With the abilitty to allocate additional buffer * headers, we can get in to the situation where "too" many * buf_bawrite()s can create situation where the kernel can create * buffers faster than the disks can service. * We limit the number of "in flight" writes a vnode can have to * avoid this. */ static int bawrite_internal(buf_t bp, int throttle) { vnode_t vp = bp->b_vp; if (vp) { if (throttle) { /* * If the vnode has "too many" write operations in progress * wait for them to finish the IO */ (void)vnode_waitforwrites(vp, VNODE_ASYNC_THROTTLE, 0, 0, (const char *)"buf_bawrite"); } else if (vp->v_numoutput >= VNODE_ASYNC_THROTTLE) { /* * return to the caller and * let him decide what to do */ return EWOULDBLOCK; } } SET(bp->b_flags, B_ASYNC); return VNOP_BWRITE(bp); } errno_t buf_bawrite(buf_t bp) { return bawrite_internal(bp, 1); } static void buf_free_meta_store(buf_t bp) { if (bp->b_bufsize) { uintptr_t datap = bp->b_datap; int bufsize = bp->b_bufsize; bp->b_datap = (uintptr_t)NULL; bp->b_bufsize = 0; /* * Ensure the assignment of b_datap has global visibility * before we free the region. */ OSMemoryBarrier(); if (ISSET(bp->b_flags, B_ZALLOC)) { kheap_free(KHEAP_VFS_BIO, datap, bufsize); } else { kmem_free(kernel_map, datap, bufsize); } } } static buf_t buf_brelse_shadow(buf_t bp) { buf_t bp_head; buf_t bp_temp; buf_t bp_return = NULL; #ifdef BUF_MAKE_PRIVATE buf_t bp_data; int data_ref = 0; #endif int need_wakeup = 0; lck_mtx_lock_spin(&buf_mtx); __IGNORE_WCASTALIGN(bp_head = (buf_t)bp->b_orig); if (bp_head->b_whichq != -1) { panic("buf_brelse_shadow: bp_head on freelist %d", bp_head->b_whichq); } #ifdef BUF_MAKE_PRIVATE if (bp_data = bp->b_data_store) { bp_data->b_data_ref--; /* * snapshot the ref count so that we can check it * outside of the lock... we only want the guy going * from 1 -> 0 to try and release the storage */ data_ref = bp_data->b_data_ref; } #endif KERNEL_DEBUG(0xbbbbc008 | DBG_FUNC_START, bp, bp_head, bp_head->b_shadow_ref, 0, 0); bp_head->b_shadow_ref--; for (bp_temp = bp_head; bp_temp && bp != bp_temp->b_shadow; bp_temp = bp_temp->b_shadow) { ; } if (bp_temp == NULL) { panic("buf_brelse_shadow: bp not on list %p", bp_head); } bp_temp->b_shadow = bp_temp->b_shadow->b_shadow; #ifdef BUF_MAKE_PRIVATE /* * we're about to free the current 'owner' of the data buffer and * there is at least one other shadow buf_t still pointing at it * so transfer it to the first shadow buf left in the chain */ if (bp == bp_data && data_ref) { if ((bp_data = bp_head->b_shadow) == NULL) { panic("buf_brelse_shadow: data_ref mismatch bp(%p)", bp); } for (bp_temp = bp_data; bp_temp; bp_temp = bp_temp->b_shadow) { bp_temp->b_data_store = bp_data; } bp_data->b_data_ref = data_ref; } #endif if (bp_head->b_shadow_ref == 0 && bp_head->b_shadow) { panic("buf_relse_shadow: b_shadow != NULL && b_shadow_ref == 0 bp(%p)", bp); } if (bp_head->b_shadow_ref && bp_head->b_shadow == 0) { panic("buf_relse_shadow: b_shadow == NULL && b_shadow_ref != 0 bp(%p)", bp); } if (bp_head->b_shadow_ref == 0) { if (!ISSET(bp_head->b_lflags, BL_BUSY)) { CLR(bp_head->b_flags, B_AGE); bp_head->b_timestamp = buf_timestamp(); if (ISSET(bp_head->b_flags, B_LOCKED)) { bp_head->b_whichq = BQ_LOCKED; binstailfree(bp_head, &bufqueues[BQ_LOCKED], BQ_LOCKED); } else { bp_head->b_whichq = BQ_META; binstailfree(bp_head, &bufqueues[BQ_META], BQ_META); } } else if (ISSET(bp_head->b_lflags, BL_WAITSHADOW)) { CLR(bp_head->b_lflags, BL_WAITSHADOW); bp_return = bp_head; } if (ISSET(bp_head->b_lflags, BL_WANTED_REF)) { CLR(bp_head->b_lflags, BL_WANTED_REF); need_wakeup = 1; } } lck_mtx_unlock(&buf_mtx); if (need_wakeup) { wakeup(bp_head); } #ifdef BUF_MAKE_PRIVATE if (bp == bp_data && data_ref == 0) { buf_free_meta_store(bp); } bp->b_data_store = NULL; #endif KERNEL_DEBUG(0xbbbbc008 | DBG_FUNC_END, bp, 0, 0, 0, 0); return bp_return; } /* * Release a buffer on to the free lists. * Described in Bach (p. 46). */ void buf_brelse(buf_t bp) { struct bqueues *bufq; int whichq; upl_t upl; int need_wakeup = 0; int need_bp_wakeup = 0; if (bp->b_whichq != -1 || !(bp->b_lflags & BL_BUSY)) { panic("buf_brelse: bad buffer = %p", bp); } #ifdef JOE_DEBUG (void) OSBacktrace(&bp->b_stackbrelse[0], 6); bp->b_lastbrelse = current_thread(); bp->b_tag = 0; #endif if (bp->b_lflags & BL_IOBUF) { buf_t shadow_master_bp = NULL; if (ISSET(bp->b_lflags, BL_SHADOW)) { shadow_master_bp = buf_brelse_shadow(bp); } else if (ISSET(bp->b_lflags, BL_IOBUF_ALLOC)) { buf_free_meta_store(bp); } free_io_buf(bp); if (shadow_master_bp) { bp = shadow_master_bp; goto finish_shadow_master; } return; } KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 388)) | DBG_FUNC_START, bp->b_lblkno * PAGE_SIZE, bp, bp->b_datap, bp->b_flags, 0); trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno); /* * if we're invalidating a buffer that has the B_FILTER bit * set then call the b_iodone function so it gets cleaned * up properly. * * the HFS journal code depends on this */ if (ISSET(bp->b_flags, B_META) && ISSET(bp->b_flags, B_INVAL)) { if (ISSET(bp->b_flags, B_FILTER)) { /* if necessary, call out */ void (*iodone_func)(struct buf *, void *) = bp->b_iodone; void *arg = bp->b_transaction; CLR(bp->b_flags, B_FILTER); /* but note callout done */ bp->b_iodone = NULL; bp->b_transaction = NULL; if (iodone_func == NULL) { panic("brelse: bp @ %p has NULL b_iodone!", bp); } (*iodone_func)(bp, arg); } } /* * I/O is done. Cleanup the UPL state */ upl = bp->b_upl; if (!ISSET(bp->b_flags, B_META) && UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) { kern_return_t kret; int upl_flags; if (upl == NULL) { if (!ISSET(bp->b_flags, B_INVAL)) { kret = ubc_create_upl_kernel(bp->b_vp, ubc_blktooff(bp->b_vp, bp->b_lblkno), bp->b_bufsize, &upl, NULL, UPL_PRECIOUS, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) { panic("brelse: Failed to create UPL"); } #if UPL_DEBUG upl_ubc_alias_set(upl, (uintptr_t) bp, (uintptr_t) 5); #endif /* UPL_DEBUG */ } } else { if (bp->b_datap) { kret = ubc_upl_unmap(upl); if (kret != KERN_SUCCESS) { panic("ubc_upl_unmap failed"); } bp->b_datap = (uintptr_t)NULL; } } if (upl) { if (bp->b_flags & (B_ERROR | B_INVAL)) { if (bp->b_flags & (B_READ | B_INVAL)) { upl_flags = UPL_ABORT_DUMP_PAGES; } else { upl_flags = 0; } ubc_upl_abort(upl, upl_flags); } else { if (ISSET(bp->b_flags, B_DELWRI | B_WASDIRTY)) { upl_flags = UPL_COMMIT_SET_DIRTY; } else { upl_flags = UPL_COMMIT_CLEAR_DIRTY; } ubc_upl_commit_range(upl, 0, bp->b_bufsize, upl_flags | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY); } bp->b_upl = NULL; } } else { if ((upl)) { panic("brelse: UPL set for non VREG; vp=%p", bp->b_vp); } } /* * If it's locked, don't report an error; try again later. */ if (ISSET(bp->b_flags, (B_LOCKED | B_ERROR)) == (B_LOCKED | B_ERROR)) { CLR(bp->b_flags, B_ERROR); } /* * If it's not cacheable, or an error, mark it invalid. */ if (ISSET(bp->b_flags, (B_NOCACHE | B_ERROR))) { SET(bp->b_flags, B_INVAL); } if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL) || (ISSET(bp->b_lflags, BL_WANTDEALLOC) && !ISSET(bp->b_flags, B_DELWRI))) { boolean_t delayed_buf_free_meta_store = FALSE; /* * If it's invalid or empty, dissociate it from its vnode, * release its storage if B_META, and * clean it up a bit and put it on the EMPTY queue */ if (ISSET(bp->b_flags, B_DELWRI)) { OSAddAtomicLong(-1, &nbdwrite); } if (ISSET(bp->b_flags, B_META)) { if (bp->b_shadow_ref) { delayed_buf_free_meta_store = TRUE; } else { buf_free_meta_store(bp); } } /* * nuke any credentials we were holding */ buf_release_credentials(bp); lck_mtx_lock_spin(&buf_mtx); if (bp->b_shadow_ref) { SET(bp->b_lflags, BL_WAITSHADOW); lck_mtx_unlock(&buf_mtx); return; } if (delayed_buf_free_meta_store == TRUE) { lck_mtx_unlock(&buf_mtx); finish_shadow_master: buf_free_meta_store(bp); lck_mtx_lock_spin(&buf_mtx); } CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA)); if (bp->b_vp) { brelvp_locked(bp); } bremhash(bp); BLISTNONE(bp); binshash(bp, &invalhash); bp->b_whichq = BQ_EMPTY; binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY); } else { /* * It has valid data. Put it on the end of the appropriate * queue, so that it'll stick around for as long as possible. */ if (ISSET(bp->b_flags, B_LOCKED)) { whichq = BQ_LOCKED; /* locked in core */ } else if (ISSET(bp->b_flags, B_META)) { whichq = BQ_META; /* meta-data */ } else if (ISSET(bp->b_flags, B_AGE)) { whichq = BQ_AGE; /* stale but valid data */ } else { whichq = BQ_LRU; /* valid data */ } bufq = &bufqueues[whichq]; bp->b_timestamp = buf_timestamp(); lck_mtx_lock_spin(&buf_mtx); /* * the buf_brelse_shadow routine doesn't take 'ownership' * of the parent buf_t... it updates state that is protected by * the buf_mtx, and checks for BL_BUSY to determine whether to * put the buf_t back on a free list. b_shadow_ref is protected * by the lock, and since we have not yet cleared B_BUSY, we need * to check it while holding the lock to insure that one of us * puts this buf_t back on a free list when it is safe to do so */ if (bp->b_shadow_ref == 0) { CLR(bp->b_flags, (B_AGE | B_ASYNC | B_NOCACHE)); bp->b_whichq = whichq; binstailfree(bp, bufq, whichq); } else { /* * there are still cloned buf_t's pointing * at this guy... need to keep it off the * freelists until a buf_brelse is done on * the last clone */ CLR(bp->b_flags, (B_ASYNC | B_NOCACHE)); } } if (needbuffer) { /* * needbuffer is a global * we're currently using buf_mtx to protect it * delay doing the actual wakeup until after * we drop buf_mtx */ needbuffer = 0; need_wakeup = 1; } if (ISSET(bp->b_lflags, BL_WANTED)) { /* * delay the actual wakeup until after we * clear BL_BUSY and we've dropped buf_mtx */ need_bp_wakeup = 1; } /* * Unlock the buffer. */ CLR(bp->b_lflags, (BL_BUSY | BL_WANTED)); buf_busycount--; lck_mtx_unlock(&buf_mtx); if (need_wakeup) { /* * Wake up any processes waiting for any buffer to become free. */ wakeup(&needbuffer); } if (need_bp_wakeup) { /* * Wake up any proceeses waiting for _this_ buffer to become free. */ wakeup(bp); } KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 388)) | DBG_FUNC_END, bp, bp->b_datap, bp->b_flags, 0, 0); } /* * Determine if a block is in the cache. * Just look on what would be its hash chain. If it's there, return * a pointer to it, unless it's marked invalid. If it's marked invalid, * we normally don't return the buffer, unless the caller explicitly * wants us to. */ static boolean_t incore(vnode_t vp, daddr64_t blkno) { boolean_t retval; struct bufhashhdr *dp; dp = BUFHASH(vp, blkno); lck_mtx_lock_spin(&buf_mtx); if (incore_locked(vp, blkno, dp)) { retval = TRUE; } else { retval = FALSE; } lck_mtx_unlock(&buf_mtx); return retval; } static buf_t incore_locked(vnode_t vp, daddr64_t blkno, struct bufhashhdr *dp) { struct buf *bp; /* Search hash chain */ for (bp = dp->lh_first; bp != NULL; bp = bp->b_hash.le_next) { if (bp->b_lblkno == blkno && bp->b_vp == vp && !ISSET(bp->b_flags, B_INVAL)) { return bp; } } return NULL; } void buf_wait_for_shadow_io(vnode_t vp, daddr64_t blkno) { buf_t bp; struct bufhashhdr *dp; dp = BUFHASH(vp, blkno); lck_mtx_lock_spin(&buf_mtx); for (;;) { if ((bp = incore_locked(vp, blkno, dp)) == NULL) { break; } if (bp->b_shadow_ref == 0) { break; } SET(bp->b_lflags, BL_WANTED_REF); (void) msleep(bp, &buf_mtx, PSPIN | (PRIBIO + 1), "buf_wait_for_shadow", NULL); } lck_mtx_unlock(&buf_mtx); } /* XXX FIXME -- Update the comment to reflect the UBC changes (please) -- */ /* * Get a block of requested size that is associated with * a given vnode and block offset. If it is found in the * block cache, mark it as having been found, make it busy * and return it. Otherwise, return an empty block of the * correct size. It is up to the caller to insure that the * cached blocks be of the correct size. */ buf_t buf_getblk(vnode_t vp, daddr64_t blkno, int size, int slpflag, int slptimeo, int operation) { buf_t bp; int err; upl_t upl; upl_page_info_t *pl; kern_return_t kret; int ret_only_valid; struct timespec ts; int upl_flags; struct bufhashhdr *dp; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_START, (uintptr_t)(blkno * PAGE_SIZE), size, operation, 0, 0); ret_only_valid = operation & BLK_ONLYVALID; operation &= ~BLK_ONLYVALID; dp = BUFHASH(vp, blkno); start: lck_mtx_lock_spin(&buf_mtx); if ((bp = incore_locked(vp, blkno, dp))) { /* * Found in the Buffer Cache */ if (ISSET(bp->b_lflags, BL_BUSY)) { /* * but is busy */ switch (operation) { case BLK_READ: case BLK_WRITE: case BLK_META: SET(bp->b_lflags, BL_WANTED); bufstats.bufs_busyincore++; /* * don't retake the mutex after being awakened... * the time out is in msecs */ ts.tv_sec = (slptimeo / 1000); ts.tv_nsec = (slptimeo % 1000) * 10 * NSEC_PER_USEC * 1000; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 396)) | DBG_FUNC_NONE, (uintptr_t)blkno, size, operation, 0, 0); err = msleep(bp, &buf_mtx, slpflag | PDROP | (PRIBIO + 1), "buf_getblk", &ts); /* * Callers who call with PCATCH or timeout are * willing to deal with the NULL pointer */ if (err && ((slpflag & PCATCH) || ((err == EWOULDBLOCK) && slptimeo))) { return NULL; } goto start; /*NOTREACHED*/ default: /* * unknown operation requested */ panic("getblk: paging or unknown operation for incore busy buffer - %x", operation); /*NOTREACHED*/ break; } } else { int clear_bdone; /* * buffer in core and not busy */ SET(bp->b_lflags, BL_BUSY); SET(bp->b_flags, B_CACHE); buf_busycount++; bremfree_locked(bp); bufstats.bufs_incore++; lck_mtx_unlock(&buf_mtx); #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 1; #endif if ((bp->b_upl)) { panic("buffer has UPL, but not marked BUSY: %p", bp); } clear_bdone = FALSE; if (!ret_only_valid) { /* * If the number bytes that are valid is going * to increase (even if we end up not doing a * reallocation through allocbuf) we have to read * the new size first. * * This is required in cases where we doing a read * modify write of a already valid data on disk but * in cases where the data on disk beyond (blkno + b_bcount) * is invalid, we may end up doing extra I/O. */ if (operation == BLK_META && bp->b_bcount < (uint32_t)size) { /* * Since we are going to read in the whole size first * we first have to ensure that any pending delayed write * is flushed to disk first. */ if (ISSET(bp->b_flags, B_DELWRI)) { CLR(bp->b_flags, B_CACHE); buf_bwrite(bp); goto start; } /* * clear B_DONE before returning from * this function so that the caller can * can issue a read for the new size. */ clear_bdone = TRUE; } if (bp->b_bufsize != (uint32_t)size) { allocbuf(bp, size); } } upl_flags = 0; switch (operation) { case BLK_WRITE: /* * "write" operation: let the UPL subsystem * know that we intend to modify the buffer * cache pages we're gathering. */ upl_flags |= UPL_WILL_MODIFY; OS_FALLTHROUGH; case BLK_READ: upl_flags |= UPL_PRECIOUS; if (UBCINFOEXISTS(bp->b_vp) && bp->b_bufsize) { kret = ubc_create_upl_kernel(vp, ubc_blktooff(vp, bp->b_lblkno), bp->b_bufsize, &upl, &pl, upl_flags, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) { panic("Failed to create UPL"); } bp->b_upl = upl; if (upl_valid_page(pl, 0)) { if (upl_dirty_page(pl, 0)) { SET(bp->b_flags, B_WASDIRTY); } else { CLR(bp->b_flags, B_WASDIRTY); } } else { CLR(bp->b_flags, (B_DONE | B_CACHE | B_WASDIRTY | B_DELWRI)); } kret = ubc_upl_map(upl, (vm_offset_t*)&(bp->b_datap)); if (kret != KERN_SUCCESS) { panic("getblk: ubc_upl_map() failed with (%d)", kret); } } break; case BLK_META: /* * VM is not involved in IO for the meta data * buffer already has valid data */ break; default: panic("getblk: paging or unknown operation for incore buffer- %d", operation); /*NOTREACHED*/ break; } if (clear_bdone) { CLR(bp->b_flags, B_DONE); } } } else { /* not incore() */ int queue = BQ_EMPTY; /* Start with no preference */ if (ret_only_valid) { lck_mtx_unlock(&buf_mtx); return NULL; } if ((vnode_isreg(vp) == 0) || (UBCINFOEXISTS(vp) == 0) /*|| (vnode_issystem(vp) == 1)*/) { operation = BLK_META; } if ((bp = getnewbuf(slpflag, slptimeo, &queue)) == NULL) { goto start; } /* * getnewbuf may block for a number of different reasons... * if it does, it's then possible for someone else to * create a buffer for the same block and insert it into * the hash... if we see it incore at this point we dump * the buffer we were working on and start over */ if (incore_locked(vp, blkno, dp)) { SET(bp->b_flags, B_INVAL); binshash(bp, &invalhash); lck_mtx_unlock(&buf_mtx); buf_brelse(bp); goto start; } /* * NOTE: YOU CAN NOT BLOCK UNTIL binshash() HAS BEEN * CALLED! BE CAREFUL. */ /* * mark the buffer as B_META if indicated * so that when buffer is released it will goto META queue */ if (operation == BLK_META) { SET(bp->b_flags, B_META); } bp->b_blkno = bp->b_lblkno = blkno; bp->b_lblksize = 0; /* Should be set by caller */ bp->b_vp = vp; /* * Insert in the hash so that incore() can find it */ binshash(bp, BUFHASH(vp, blkno)); bgetvp_locked(vp, bp); lck_mtx_unlock(&buf_mtx); allocbuf(bp, size); upl_flags = 0; switch (operation) { case BLK_META: /* * buffer data is invalid... * * I don't want to have to retake buf_mtx, * so the miss and vmhits counters are done * with Atomic updates... all other counters * in bufstats are protected with either * buf_mtx or iobuffer_mtxp */ OSAddAtomicLong(1, &bufstats.bufs_miss); break; case BLK_WRITE: /* * "write" operation: let the UPL subsystem know * that we intend to modify the buffer cache pages * we're gathering. */ upl_flags |= UPL_WILL_MODIFY; OS_FALLTHROUGH; case BLK_READ: { off_t f_offset; size_t contig_bytes; int bmap_flags; #if DEVELOPMENT || DEBUG /* * Apple implemented file systems use UBC excludively; they should * not call in here." */ const char* excldfs[] = {"hfs", "afpfs", "smbfs", "acfs", "exfat", "msdos", "webdav", NULL}; for (int i = 0; excldfs[i] != NULL; i++) { if (vp->v_mount && !strcmp(vp->v_mount->mnt_vfsstat.f_fstypename, excldfs[i])) { panic("%s %s calls buf_getblk", excldfs[i], operation == BLK_READ ? "BLK_READ" : "BLK_WRITE"); } } #endif if ((bp->b_upl)) { panic("bp already has UPL: %p", bp); } f_offset = ubc_blktooff(vp, blkno); upl_flags |= UPL_PRECIOUS; kret = ubc_create_upl_kernel(vp, f_offset, bp->b_bufsize, &upl, &pl, upl_flags, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) { panic("Failed to create UPL"); } #if UPL_DEBUG upl_ubc_alias_set(upl, (uintptr_t) bp, (uintptr_t) 4); #endif /* UPL_DEBUG */ bp->b_upl = upl; if (upl_valid_page(pl, 0)) { if (operation == BLK_READ) { bmap_flags = VNODE_READ; } else { bmap_flags = VNODE_WRITE; } SET(bp->b_flags, B_CACHE | B_DONE); OSAddAtomicLong(1, &bufstats.bufs_vmhits); bp->b_validoff = 0; bp->b_dirtyoff = 0; if (upl_dirty_page(pl, 0)) { /* page is dirty */ SET(bp->b_flags, B_WASDIRTY); bp->b_validend = bp->b_bcount; bp->b_dirtyend = bp->b_bcount; } else { /* page is clean */ bp->b_validend = bp->b_bcount; bp->b_dirtyend = 0; } /* * try to recreate the physical block number associated with * this buffer... */ if (VNOP_BLOCKMAP(vp, f_offset, bp->b_bcount, &bp->b_blkno, &contig_bytes, NULL, bmap_flags, NULL)) { panic("getblk: VNOP_BLOCKMAP failed"); } /* * if the extent represented by this buffer * is not completely physically contiguous on * disk, than we can't cache the physical mapping * in the buffer header */ if ((uint32_t)contig_bytes < bp->b_bcount) { bp->b_blkno = bp->b_lblkno; } } else { OSAddAtomicLong(1, &bufstats.bufs_miss); } kret = ubc_upl_map(upl, (vm_offset_t *)&(bp->b_datap)); if (kret != KERN_SUCCESS) { panic("getblk: ubc_upl_map() failed with (%d)", kret); } break;} // end BLK_READ default: panic("getblk: paging or unknown operation - %x", operation); /*NOTREACHED*/ break; } // end switch } //end buf_t !incore KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_END, bp, bp->b_datap, bp->b_flags, 3, 0); #ifdef JOE_DEBUG (void) OSBacktrace(&bp->b_stackgetblk[0], 6); #endif return bp; } /* * Get an empty, disassociated buffer of given size. */ buf_t buf_geteblk(int size) { buf_t bp = NULL; int queue = BQ_EMPTY; do { lck_mtx_lock_spin(&buf_mtx); bp = getnewbuf(0, 0, &queue); } while (bp == NULL); SET(bp->b_flags, (B_META | B_INVAL)); #if DIAGNOSTIC assert(queue == BQ_EMPTY); #endif /* DIAGNOSTIC */ /* XXX need to implement logic to deal with other queues */ binshash(bp, &invalhash); bufstats.bufs_eblk++; lck_mtx_unlock(&buf_mtx); allocbuf(bp, size); return bp; } uint32_t buf_redundancy_flags(buf_t bp) { return bp->b_redundancy_flags; } void buf_set_redundancy_flags(buf_t bp, uint32_t flags) { SET(bp->b_redundancy_flags, flags); } void buf_clear_redundancy_flags(buf_t bp, uint32_t flags) { CLR(bp->b_redundancy_flags, flags); } static void * recycle_buf_from_pool(int nsize) { buf_t bp; void *ptr = NULL; lck_mtx_lock_spin(&buf_mtx); TAILQ_FOREACH(bp, &bufqueues[BQ_META], b_freelist) { if (ISSET(bp->b_flags, B_DELWRI) || bp->b_bufsize != (uint32_t)nsize) { continue; } ptr = (void *)bp->b_datap; bp->b_bufsize = 0; bcleanbuf(bp, TRUE); break; } lck_mtx_unlock(&buf_mtx); return ptr; } int zalloc_nopagewait_failed = 0; int recycle_buf_failed = 0; static void * grab_memory_for_meta_buf(int nsize) { void *ptr; boolean_t was_vmpriv; /* * make sure we're NOT priviliged so that * if a vm_page_grab is needed, it won't * block if we're out of free pages... if * it blocks, then we can't honor the * nopagewait request */ was_vmpriv = set_vm_privilege(FALSE); ptr = kheap_alloc(KHEAP_VFS_BIO, nsize, Z_NOPAGEWAIT); if (was_vmpriv == TRUE) { set_vm_privilege(TRUE); } if (ptr == NULL) { zalloc_nopagewait_failed++; ptr = recycle_buf_from_pool(nsize); if (ptr == NULL) { recycle_buf_failed++; if (was_vmpriv == FALSE) { set_vm_privilege(TRUE); } ptr = kheap_alloc(KHEAP_VFS_BIO, nsize, Z_WAITOK); if (was_vmpriv == FALSE) { set_vm_privilege(FALSE); } } } return ptr; } /* * With UBC, there is no need to expand / shrink the file data * buffer. The VM uses the same pages, hence no waste. * All the file data buffers can have one size. * In fact expand / shrink would be an expensive operation. * * Only exception to this is meta-data buffers. Most of the * meta data operations are smaller than PAGE_SIZE. Having the * meta-data buffers grow and shrink as needed, optimizes use * of the kernel wired memory. */ int allocbuf(buf_t bp, int size) { vm_size_t desired_size; desired_size = roundup(size, CLBYTES); if (desired_size < PAGE_SIZE) { desired_size = PAGE_SIZE; } if (desired_size > MAXBSIZE) { panic("allocbuf: buffer larger than MAXBSIZE requested"); } if (ISSET(bp->b_flags, B_META)) { int nsize = roundup(size, MINMETA); if (bp->b_datap) { void *elem = (void *)bp->b_datap; if (ISSET(bp->b_flags, B_ZALLOC)) { if (bp->b_bufsize < (uint32_t)nsize) { /* reallocate to a bigger size */ if (nsize <= MAXMETA) { desired_size = nsize; /* b_datap not really a ptr */ *(void **)(&bp->b_datap) = grab_memory_for_meta_buf(nsize); } else { bp->b_datap = (uintptr_t)NULL; kmem_alloc(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size, KMA_KOBJECT | KMA_DATA | KMA_NOFAIL, VM_KERN_MEMORY_FILE); CLR(bp->b_flags, B_ZALLOC); } bcopy(elem, (caddr_t)bp->b_datap, bp->b_bufsize); kheap_free(KHEAP_VFS_BIO, elem, bp->b_bufsize); } else { desired_size = bp->b_bufsize; } } else { if ((vm_size_t)bp->b_bufsize < desired_size) { /* reallocate to a bigger size */ bp->b_datap = (uintptr_t)NULL; kmem_alloc(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size, KMA_KOBJECT | KMA_DATA | KMA_NOFAIL, VM_KERN_MEMORY_FILE); bcopy(elem, (caddr_t)bp->b_datap, bp->b_bufsize); kmem_free(kernel_map, (vm_offset_t)elem, bp->b_bufsize); } else { desired_size = bp->b_bufsize; } } } else { /* new allocation */ if (nsize <= MAXMETA) { desired_size = nsize; /* b_datap not really a ptr */ *(void **)(&bp->b_datap) = grab_memory_for_meta_buf(nsize); SET(bp->b_flags, B_ZALLOC); } else { kmem_alloc(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size, KMA_KOBJECT | KMA_DATA | KMA_NOFAIL, VM_KERN_MEMORY_FILE); } } } bp->b_bufsize = (uint32_t)desired_size; bp->b_bcount = size; return 0; } /* * Get a new buffer from one of the free lists. * * Request for a queue is passes in. The queue from which the buffer was taken * from is returned. Out of range queue requests get BQ_EMPTY. Request for * BQUEUE means no preference. Use heuristics in that case. * Heuristics is as follows: * Try BQ_AGE, BQ_LRU, BQ_EMPTY, BQ_META in that order. * If none available block till one is made available. * If buffers available on both BQ_AGE and BQ_LRU, check the timestamps. * Pick the most stale buffer. * If found buffer was marked delayed write, start the async. write * and restart the search. * Initialize the fields and disassociate the buffer from the vnode. * Remove the buffer from the hash. Return the buffer and the queue * on which it was found. * * buf_mtx is held upon entry * returns with buf_mtx locked if new buf available * returns with buf_mtx UNlocked if new buf NOT available */ static buf_t getnewbuf(int slpflag, int slptimeo, int * queue) { buf_t bp; buf_t lru_bp; buf_t age_bp; buf_t meta_bp; int age_time, lru_time, bp_time, meta_time; int req = *queue; /* save it for restarts */ struct timespec ts; start: /* * invalid request gets empty queue */ if ((*queue >= BQUEUES) || (*queue < 0) || (*queue == BQ_LAUNDRY) || (*queue == BQ_LOCKED)) { *queue = BQ_EMPTY; } if (*queue == BQ_EMPTY && (bp = bufqueues[*queue].tqh_first)) { goto found; } /* * need to grow number of bufs, add another one rather than recycling */ if (nbuf_headers < max_nbuf_headers) { /* * Increment count now as lock * is dropped for allocation. * That avoids over commits */ nbuf_headers++; goto add_newbufs; } /* Try for the requested queue first */ bp = bufqueues[*queue].tqh_first; if (bp) { goto found; } /* Unable to use requested queue */ age_bp = bufqueues[BQ_AGE].tqh_first; lru_bp = bufqueues[BQ_LRU].tqh_first; meta_bp = bufqueues[BQ_META].tqh_first; if (!age_bp && !lru_bp && !meta_bp) { /* * Unavailble on AGE or LRU or META queues * Try the empty list first */ bp = bufqueues[BQ_EMPTY].tqh_first; if (bp) { *queue = BQ_EMPTY; goto found; } /* * We have seen is this is hard to trigger. * This is an overcommit of nbufs but needed * in some scenarios with diskiamges */ add_newbufs: lck_mtx_unlock(&buf_mtx); /* Create a new temporary buffer header */ bp = zalloc_flags(buf_hdr_zone, Z_WAITOK | Z_NOFAIL); bufhdrinit(bp); bp->b_whichq = BQ_EMPTY; bp->b_timestamp = buf_timestamp(); BLISTNONE(bp); SET(bp->b_flags, B_HDRALLOC); *queue = BQ_EMPTY; lck_mtx_lock_spin(&buf_mtx); if (bp) { binshash(bp, &invalhash); binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY); buf_hdr_count++; goto found; } /* subtract already accounted bufcount */ nbuf_headers--; bufstats.bufs_sleeps++; /* wait for a free buffer of any kind */ needbuffer = 1; /* hz value is 100 */ ts.tv_sec = (slptimeo / 1000); /* the hz value is 100; which leads to 10ms */ ts.tv_nsec = (slptimeo % 1000) * NSEC_PER_USEC * 1000 * 10; msleep(&needbuffer, &buf_mtx, slpflag | PDROP | (PRIBIO + 1), "getnewbuf", &ts); return NULL; } /* Buffer available either on AGE or LRU or META */ bp = NULL; *queue = -1; /* Buffer available either on AGE or LRU */ if (!age_bp) { bp = lru_bp; *queue = BQ_LRU; } else if (!lru_bp) { bp = age_bp; *queue = BQ_AGE; } else { /* buffer available on both AGE and LRU */ int t = buf_timestamp(); age_time = t - age_bp->b_timestamp; lru_time = t - lru_bp->b_timestamp; if ((age_time < 0) || (lru_time < 0)) { /* time set backwards */ bp = age_bp; *queue = BQ_AGE; /* * we should probably re-timestamp eveything in the * queues at this point with the current time */ } else { if ((lru_time >= lru_is_stale) && (age_time < age_is_stale)) { bp = lru_bp; *queue = BQ_LRU; } else { bp = age_bp; *queue = BQ_AGE; } } } if (!bp) { /* Neither on AGE nor on LRU */ bp = meta_bp; *queue = BQ_META; } else if (meta_bp) { int t = buf_timestamp(); bp_time = t - bp->b_timestamp; meta_time = t - meta_bp->b_timestamp; if (!(bp_time < 0) && !(meta_time < 0)) { /* time not set backwards */ int bp_is_stale; bp_is_stale = (*queue == BQ_LRU) ? lru_is_stale : age_is_stale; if ((meta_time >= meta_is_stale) && (bp_time < bp_is_stale)) { bp = meta_bp; *queue = BQ_META; } } } found: if (ISSET(bp->b_flags, B_LOCKED) || ISSET(bp->b_lflags, BL_BUSY)) { panic("getnewbuf: bp @ %p is LOCKED or BUSY! (flags 0x%x)", bp, bp->b_flags); } /* Clean it */ if (bcleanbuf(bp, FALSE)) { /* * moved to the laundry thread, buffer not ready */ *queue = req; goto start; } return bp; } /* * Clean a buffer. * Returns 0 if buffer is ready to use, * Returns 1 if issued a buf_bawrite() to indicate * that the buffer is not ready. * * buf_mtx is held upon entry * returns with buf_mtx locked */ int bcleanbuf(buf_t bp, boolean_t discard) { /* Remove from the queue */ bremfree_locked(bp); #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 2; #endif /* * If buffer was a delayed write, start the IO by queuing * it on the LAUNDRY queue, and return 1 */ if (ISSET(bp->b_flags, B_DELWRI)) { if (discard) { SET(bp->b_lflags, BL_WANTDEALLOC); } bmovelaundry(bp); lck_mtx_unlock(&buf_mtx); wakeup(&bufqueues[BQ_LAUNDRY]); /* * and give it a chance to run */ (void)thread_block(THREAD_CONTINUE_NULL); lck_mtx_lock_spin(&buf_mtx); return 1; } #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 8; #endif /* * Buffer is no longer on any free list... we own it */ SET(bp->b_lflags, BL_BUSY); buf_busycount++; bremhash(bp); /* * disassociate us from our vnode, if we had one... */ if (bp->b_vp) { brelvp_locked(bp); } lck_mtx_unlock(&buf_mtx); BLISTNONE(bp); if (ISSET(bp->b_flags, B_META)) { buf_free_meta_store(bp); } trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno); buf_release_credentials(bp); /* If discarding, just move to the empty queue */ if (discard) { lck_mtx_lock_spin(&buf_mtx); CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA)); bp->b_whichq = BQ_EMPTY; binshash(bp, &invalhash); binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY); CLR(bp->b_lflags, BL_BUSY); buf_busycount--; } else { /* Not discarding: clean up and prepare for reuse */ bp->b_bufsize = 0; bp->b_datap = (uintptr_t)NULL; bp->b_upl = (void *)NULL; bp->b_fsprivate = (void *)NULL; /* * preserve the state of whether this buffer * was allocated on the fly or not... * the only other flag that should be set at * this point is BL_BUSY... */ #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 3; #endif bp->b_lflags = BL_BUSY; bp->b_flags = (bp->b_flags & B_HDRALLOC); bp->b_redundancy_flags = 0; bp->b_dev = NODEV; bp->b_blkno = bp->b_lblkno = 0; bp->b_lblksize = 0; bp->b_iodone = NULL; bp->b_error = 0; bp->b_resid = 0; bp->b_bcount = 0; bp->b_dirtyoff = bp->b_dirtyend = 0; bp->b_validoff = bp->b_validend = 0; bzero(&bp->b_attr, sizeof(struct bufattr)); lck_mtx_lock_spin(&buf_mtx); } return 0; } errno_t buf_invalblkno(vnode_t vp, daddr64_t lblkno, int flags) { buf_t bp; errno_t error; struct bufhashhdr *dp; dp = BUFHASH(vp, lblkno); relook: lck_mtx_lock_spin(&buf_mtx); if ((bp = incore_locked(vp, lblkno, dp)) == (struct buf *)0) { lck_mtx_unlock(&buf_mtx); return 0; } if (ISSET(bp->b_lflags, BL_BUSY)) { if (!ISSET(flags, BUF_WAIT)) { lck_mtx_unlock(&buf_mtx); return EBUSY; } SET(bp->b_lflags, BL_WANTED); error = msleep((caddr_t)bp, &buf_mtx, PDROP | (PRIBIO + 1), "buf_invalblkno", NULL); if (error) { return error; } goto relook; } bremfree_locked(bp); SET(bp->b_lflags, BL_BUSY); SET(bp->b_flags, B_INVAL); buf_busycount++; #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 4; #endif lck_mtx_unlock(&buf_mtx); buf_brelse(bp); return 0; } void buf_drop(buf_t bp) { int need_wakeup = 0; lck_mtx_lock_spin(&buf_mtx); if (ISSET(bp->b_lflags, BL_WANTED)) { /* * delay the actual wakeup until after we * clear BL_BUSY and we've dropped buf_mtx */ need_wakeup = 1; } #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 9; #endif /* * Unlock the buffer. */ CLR(bp->b_lflags, (BL_BUSY | BL_WANTED)); buf_busycount--; lck_mtx_unlock(&buf_mtx); if (need_wakeup) { /* * Wake up any proceeses waiting for _this_ buffer to become free. */ wakeup(bp); } } errno_t buf_acquire(buf_t bp, int flags, int slpflag, int slptimeo) { errno_t error; lck_mtx_lock_spin(&buf_mtx); error = buf_acquire_locked(bp, flags, slpflag, slptimeo); lck_mtx_unlock(&buf_mtx); return error; } static errno_t buf_acquire_locked(buf_t bp, int flags, int slpflag, int slptimeo) { errno_t error; struct timespec ts; if (ISSET(bp->b_flags, B_LOCKED)) { if ((flags & BAC_SKIP_LOCKED)) { return EDEADLK; } } else { if ((flags & BAC_SKIP_NONLOCKED)) { return EDEADLK; } } if (ISSET(bp->b_lflags, BL_BUSY)) { /* * since the lck_mtx_lock may block, the buffer * may become BUSY, so we need to * recheck for a NOWAIT request */ if (flags & BAC_NOWAIT) { return EBUSY; } SET(bp->b_lflags, BL_WANTED); /* the hz value is 100; which leads to 10ms */ ts.tv_sec = (slptimeo / 100); ts.tv_nsec = (slptimeo % 100) * 10 * NSEC_PER_USEC * 1000; error = msleep((caddr_t)bp, &buf_mtx, slpflag | (PRIBIO + 1), "buf_acquire", &ts); if (error) { return error; } return EAGAIN; } if (flags & BAC_REMOVE) { bremfree_locked(bp); } SET(bp->b_lflags, BL_BUSY); buf_busycount++; #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 5; #endif return 0; } /* * Wait for operations on the buffer to complete. * When they do, extract and return the I/O's error value. */ errno_t buf_biowait(buf_t bp) { while (!ISSET(bp->b_flags, B_DONE)) { lck_mtx_lock_spin(&buf_mtx); if (!ISSET(bp->b_flags, B_DONE)) { DTRACE_IO1(wait__start, buf_t, bp); (void) msleep(bp, &buf_mtx, PDROP | (PRIBIO + 1), "buf_biowait", NULL); DTRACE_IO1(wait__done, buf_t, bp); } else { lck_mtx_unlock(&buf_mtx); } } /* check for interruption of I/O (e.g. via NFS), then errors. */ if (ISSET(bp->b_flags, B_EINTR)) { CLR(bp->b_flags, B_EINTR); return EINTR; } else if (ISSET(bp->b_flags, B_ERROR)) { return bp->b_error ? bp->b_error : EIO; } else { return 0; } } /* * Mark I/O complete on a buffer. * * If a callback has been requested, e.g. the pageout * daemon, do so. Otherwise, awaken waiting processes. * * [ Leffler, et al., says on p.247: * "This routine wakes up the blocked process, frees the buffer * for an asynchronous write, or, for a request by the pagedaemon * process, invokes a procedure specified in the buffer structure" ] * * In real life, the pagedaemon (or other system processes) wants * to do async stuff to, and doesn't want the buffer buf_brelse()'d. * (for swap pager, that puts swap buffers on the free lists (!!!), * for the vn device, that puts malloc'd buffers on the free lists!) */ void buf_biodone(buf_t bp) { mount_t mp; struct bufattr *bap; struct timeval real_elapsed; uint64_t real_elapsed_usec = 0; KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_START, bp, bp->b_datap, bp->b_flags, 0, 0); /* Record our progress. */ vfs_update_last_completion_time(); if (ISSET(bp->b_flags, B_DONE)) { panic("biodone already"); } bap = &bp->b_attr; if (bp->b_vp && bp->b_vp->v_mount) { mp = bp->b_vp->v_mount; } else { mp = NULL; } if (ISSET(bp->b_flags, B_ERROR)) { if (mp && (MNT_ROOTFS & mp->mnt_flag)) { dk_error_description_t desc; bzero(&desc, sizeof(desc)); desc.description = panic_disk_error_description; desc.description_size = panic_disk_error_description_size; VNOP_IOCTL(mp->mnt_devvp, DKIOCGETERRORDESCRIPTION, (caddr_t)&desc, 0, vfs_context_kernel()); } } if (mp && (bp->b_flags & B_READ) == 0) { update_last_io_time(mp); INCR_PENDING_IO(-(pending_io_t)buf_count(bp), mp->mnt_pending_write_size); } else if (mp) { INCR_PENDING_IO(-(pending_io_t)buf_count(bp), mp->mnt_pending_read_size); } throttle_info_end_io(bp); if (kdebug_enable) { int code = DKIO_DONE; int io_tier = GET_BUFATTR_IO_TIER(bap); if (bp->b_flags & B_READ) { code |= DKIO_READ; } if (bp->b_flags & B_ASYNC) { code |= DKIO_ASYNC; } if (bp->b_flags & B_META) { code |= DKIO_META; } else if (bp->b_flags & B_PAGEIO) { code |= DKIO_PAGING; } if (io_tier != 0) { code |= DKIO_THROTTLE; } code |= ((io_tier << DKIO_TIER_SHIFT) & DKIO_TIER_MASK); if (bp->b_flags & B_PASSIVE) { code |= DKIO_PASSIVE; } if (bap->ba_flags & BA_NOCACHE) { code |= DKIO_NOCACHE; } if (bap->ba_flags & BA_IO_TIER_UPGRADE) { code |= DKIO_TIER_UPGRADE; } KDBG_RELEASE_NOPROCFILT(FSDBG_CODE(DBG_DKRW, code), buf_kernel_addrperm_addr(bp), (uintptr_t)VM_KERNEL_ADDRPERM(bp->b_vp), bp->b_resid, bp->b_error); } microuptime(&real_elapsed); timevalsub(&real_elapsed, &bp->b_timestamp_tv); real_elapsed_usec = real_elapsed.tv_sec * USEC_PER_SEC + real_elapsed.tv_usec; disk_conditioner_delay(bp, 1, bp->b_bcount, real_elapsed_usec); /* * I/O was done, so don't believe * the DIRTY state from VM anymore... * and we need to reset the THROTTLED/PASSIVE * indicators */ CLR(bp->b_flags, (B_WASDIRTY | B_PASSIVE)); CLR(bap->ba_flags, (BA_META | BA_NOCACHE | BA_DELAYIDLESLEEP | BA_IO_TIER_UPGRADE)); SET_BUFATTR_IO_TIER(bap, 0); DTRACE_IO1(done, buf_t, bp); if (!ISSET(bp->b_flags, B_READ) && !ISSET(bp->b_flags, B_RAW)) { /* * wake up any writer's blocked * on throttle or waiting for I/O * to drain */ vnode_writedone(bp->b_vp); } if (ISSET(bp->b_flags, (B_CALL | B_FILTER))) { /* if necessary, call out */ void (*iodone_func)(struct buf *, void *) = bp->b_iodone; void *arg = bp->b_transaction; int callout = ISSET(bp->b_flags, B_CALL); if (iodone_func == NULL) { panic("biodone: bp @ %p has NULL b_iodone!", bp); } CLR(bp->b_flags, (B_CALL | B_FILTER)); /* filters and callouts are one-shot */ bp->b_iodone = NULL; bp->b_transaction = NULL; if (callout) { SET(bp->b_flags, B_DONE); /* note that it's done */ } (*iodone_func)(bp, arg); if (callout) { /* * assumes that the callback function takes * ownership of the bp and deals with releasing it if necessary */ goto biodone_done; } /* * in this case the call back function is acting * strictly as a filter... it does not take * ownership of the bp and is expecting us * to finish cleaning up... this is currently used * by the HFS journaling code */ } if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release it */ SET(bp->b_flags, B_DONE); /* note that it's done */ buf_brelse(bp); } else { /* or just wakeup the buffer */ /* * by taking the mutex, we serialize * the buf owner calling buf_biowait so that we'll * only see him in one of 2 states... * state 1: B_DONE wasn't set and he's * blocked in msleep * state 2: he's blocked trying to take the * mutex before looking at B_DONE * BL_WANTED is cleared in case anyone else * is blocked waiting for the buffer... note * that we haven't cleared B_BUSY yet, so if * they do get to run, their going to re-set * BL_WANTED and go back to sleep */ lck_mtx_lock_spin(&buf_mtx); CLR(bp->b_lflags, BL_WANTED); SET(bp->b_flags, B_DONE); /* note that it's done */ lck_mtx_unlock(&buf_mtx); wakeup(bp); } biodone_done: KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 387)) | DBG_FUNC_END, (uintptr_t)bp, (uintptr_t)bp->b_datap, bp->b_flags, 0, 0); } /* * Obfuscate buf pointers. */ vm_offset_t buf_kernel_addrperm_addr(void * addr) { if ((vm_offset_t)addr == 0) { return 0; } else { return (vm_offset_t)addr + buf_kernel_addrperm; } } /* * Return a count of buffers on the "locked" queue. */ int count_lock_queue(void) { buf_t bp; int n = 0; lck_mtx_lock_spin(&buf_mtx); for (bp = bufqueues[BQ_LOCKED].tqh_first; bp; bp = bp->b_freelist.tqe_next) { n++; } lck_mtx_unlock(&buf_mtx); return n; } /* * Return a count of 'busy' buffers. Used at the time of shutdown. * note: This is also called from the mach side in debug context in kdp.c */ uint32_t count_busy_buffers(void) { return buf_busycount + bufstats.bufs_iobufinuse; } #if DIAGNOSTIC /* * Print out statistics on the current allocation of the buffer pool. * Can be enabled to print out on every ``sync'' by setting "syncprt" * in vfs_syscalls.c using sysctl. */ void vfs_bufstats() { int i, j, count; struct buf *bp; struct bqueues *dp; int counts[MAXBSIZE / CLBYTES + 1]; static char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE", "EMPTY", "META", "LAUNDRY" }; for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) { count = 0; for (j = 0; j <= MAXBSIZE / CLBYTES; j++) { counts[j] = 0; } lck_mtx_lock(&buf_mtx); for (bp = dp->tqh_first; bp; bp = bp->b_freelist.tqe_next) { counts[bp->b_bufsize / CLBYTES]++; count++; } lck_mtx_unlock(&buf_mtx); printf("%s: total-%d", bname[i], count); for (j = 0; j <= MAXBSIZE / CLBYTES; j++) { if (counts[j] != 0) { printf(", %d-%d", j * CLBYTES, counts[j]); } } printf("\n"); } } #endif /* DIAGNOSTIC */ #define NRESERVEDIOBUFS 128 #define MNT_VIRTUALDEV_MAX_IOBUFS 128 #define VIRTUALDEV_MAX_IOBUFS ((40*niobuf_headers)/100) buf_t alloc_io_buf(vnode_t vp, int priv) { buf_t bp; mount_t mp = NULL; int alloc_for_virtualdev = FALSE; lck_mtx_lock_spin(&iobuffer_mtxp); /* * We subject iobuf requests for diskimages to additional restrictions. * * a) A single diskimage mount cannot use up more than * MNT_VIRTUALDEV_MAX_IOBUFS. However,vm privileged (pageout) requests * are not subject to this restriction. * b) iobuf headers used by all diskimage headers by all mount * points cannot exceed VIRTUALDEV_MAX_IOBUFS. */ if (vp && ((mp = vp->v_mount)) && mp != dead_mountp && mp->mnt_kern_flag & MNTK_VIRTUALDEV) { alloc_for_virtualdev = TRUE; while ((!priv && mp->mnt_iobufinuse > MNT_VIRTUALDEV_MAX_IOBUFS) || bufstats.bufs_iobufinuse_vdev > VIRTUALDEV_MAX_IOBUFS) { bufstats.bufs_iobufsleeps++; need_iobuffer = 1; (void)msleep(&need_iobuffer, &iobuffer_mtxp, PSPIN | (PRIBIO + 1), (const char *)"alloc_io_buf (1)", NULL); } } while ((((uint32_t)(niobuf_headers - NRESERVEDIOBUFS) < bufstats.bufs_iobufinuse) && !priv) || (bp = iobufqueue.tqh_first) == NULL) { bufstats.bufs_iobufsleeps++; need_iobuffer = 1; (void)msleep(&need_iobuffer, &iobuffer_mtxp, PSPIN | (PRIBIO + 1), (const char *)"alloc_io_buf (2)", NULL); } TAILQ_REMOVE(&iobufqueue, bp, b_freelist); bufstats.bufs_iobufinuse++; if (bufstats.bufs_iobufinuse > bufstats.bufs_iobufmax) { bufstats.bufs_iobufmax = bufstats.bufs_iobufinuse; } if (alloc_for_virtualdev) { mp->mnt_iobufinuse++; bufstats.bufs_iobufinuse_vdev++; } lck_mtx_unlock(&iobuffer_mtxp); /* * initialize various fields * we don't need to hold the mutex since the buffer * is now private... the vp should have a reference * on it and is not protected by this mutex in any event */ bp->b_timestamp = 0; bp->b_proc = NULL; bp->b_datap = 0; bp->b_flags = 0; bp->b_lflags = BL_BUSY | BL_IOBUF; if (alloc_for_virtualdev) { bp->b_lflags |= BL_IOBUF_VDEV; } bp->b_redundancy_flags = 0; bp->b_blkno = bp->b_lblkno = 0; bp->b_lblksize = 0; #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 6; #endif bp->b_iodone = NULL; bp->b_error = 0; bp->b_resid = 0; bp->b_bcount = 0; bp->b_bufsize = 0; bp->b_upl = NULL; bp->b_fsprivate = (void *)NULL; bp->b_vp = vp; bzero(&bp->b_attr, sizeof(struct bufattr)); if (vp && (vp->v_type == VBLK || vp->v_type == VCHR)) { bp->b_dev = vp->v_rdev; } else { bp->b_dev = NODEV; } return bp; } void free_io_buf(buf_t bp) { int need_wakeup = 0; int free_for_virtualdev = FALSE; mount_t mp = NULL; /* Was this iobuf for a diskimage ? */ if (bp->b_lflags & BL_IOBUF_VDEV) { free_for_virtualdev = TRUE; if (bp->b_vp) { mp = bp->b_vp->v_mount; } } /* * put buffer back on the head of the iobufqueue */ bp->b_vp = NULL; bp->b_flags = B_INVAL; /* Zero out the bufattr and its flags before relinquishing this iobuf */ bzero(&bp->b_attr, sizeof(struct bufattr)); lck_mtx_lock_spin(&iobuffer_mtxp); binsheadfree(bp, &iobufqueue, -1); if (need_iobuffer) { /* * Wake up any processes waiting because they need an io buffer * * do the wakeup after we drop the mutex... it's possible that the * wakeup will be superfluous if need_iobuffer gets set again and * another thread runs this path, but it's highly unlikely, doesn't * hurt, and it means we don't hold up I/O progress if the wakeup blocks * trying to grab a task related lock... */ need_iobuffer = 0; need_wakeup = 1; } if (bufstats.bufs_iobufinuse <= 0) { panic("free_io_buf: bp(%p) - bufstats.bufs_iobufinuse < 0", bp); } bufstats.bufs_iobufinuse--; if (free_for_virtualdev) { bufstats.bufs_iobufinuse_vdev--; if (mp && mp != dead_mountp) { mp->mnt_iobufinuse--; } } lck_mtx_unlock(&iobuffer_mtxp); if (need_wakeup) { wakeup(&need_iobuffer); } } void buf_list_lock(void) { lck_mtx_lock_spin(&buf_mtx); } void buf_list_unlock(void) { lck_mtx_unlock(&buf_mtx); } /* * If getnewbuf() calls bcleanbuf() on the same thread * there is a potential for stack overrun and deadlocks. * So we always handoff the work to a worker thread for completion */ static void bcleanbuf_thread_init(void) { thread_t thread = THREAD_NULL; /* create worker thread */ kernel_thread_start((thread_continue_t)bcleanbuf_thread, NULL, &thread); thread_deallocate(thread); } typedef int (*bcleanbufcontinuation)(int); __attribute__((noreturn)) static void bcleanbuf_thread(void) { struct buf *bp; int error = 0; int loopcnt = 0; for (;;) { lck_mtx_lock_spin(&buf_mtx); while ((bp = TAILQ_FIRST(&bufqueues[BQ_LAUNDRY])) == NULL) { (void)msleep0(&bufqueues[BQ_LAUNDRY], &buf_mtx, PRIBIO | PDROP, "blaundry", 0, (bcleanbufcontinuation)bcleanbuf_thread); } /* * Remove from the queue */ bremfree_locked(bp); /* * Buffer is no longer on any free list */ SET(bp->b_lflags, BL_BUSY); buf_busycount++; #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 10; #endif lck_mtx_unlock(&buf_mtx); /* * do the IO */ error = bawrite_internal(bp, 0); if (error) { bp->b_whichq = BQ_LAUNDRY; bp->b_timestamp = buf_timestamp(); lck_mtx_lock_spin(&buf_mtx); binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY); blaundrycnt++; /* we never leave a busy page on the laundry queue */ CLR(bp->b_lflags, BL_BUSY); buf_busycount--; #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 11; #endif lck_mtx_unlock(&buf_mtx); if (loopcnt > MAXLAUNDRY) { /* * bawrite_internal() can return errors if we're throttled. If we've * done several I/Os and failed, give the system some time to unthrottle * the vnode */ (void)tsleep((void *)&bufqueues[BQ_LAUNDRY], PRIBIO, "blaundry", 1); loopcnt = 0; } else { /* give other threads a chance to run */ (void)thread_block(THREAD_CONTINUE_NULL); loopcnt++; } } } } static int brecover_data(buf_t bp) { int upl_offset; upl_t upl; upl_page_info_t *pl; kern_return_t kret; vnode_t vp = bp->b_vp; int upl_flags; if (!UBCINFOEXISTS(vp) || bp->b_bufsize == 0) { goto dump_buffer; } upl_flags = UPL_PRECIOUS; if (!(buf_flags(bp) & B_READ)) { /* * "write" operation: let the UPL subsystem know * that we intend to modify the buffer cache pages we're * gathering. */ upl_flags |= UPL_WILL_MODIFY; } kret = ubc_create_upl_kernel(vp, ubc_blktooff(vp, bp->b_lblkno), bp->b_bufsize, &upl, &pl, upl_flags, VM_KERN_MEMORY_FILE); if (kret != KERN_SUCCESS) { panic("Failed to create UPL"); } for (upl_offset = 0; (uint32_t)upl_offset < bp->b_bufsize; upl_offset += PAGE_SIZE) { if (!upl_valid_page(pl, upl_offset / PAGE_SIZE) || !upl_dirty_page(pl, upl_offset / PAGE_SIZE)) { ubc_upl_abort(upl, 0); goto dump_buffer; } } bp->b_upl = upl; kret = ubc_upl_map(upl, (vm_offset_t *)&(bp->b_datap)); if (kret != KERN_SUCCESS) { panic("getblk: ubc_upl_map() failed with (%d)", kret); } return 1; dump_buffer: bp->b_bufsize = 0; SET(bp->b_flags, B_INVAL); buf_brelse(bp); return 0; } int fs_buffer_cache_gc_register(void (* callout)(int, void *), void *context) { lck_mtx_lock(&buf_gc_callout); for (int i = 0; i < FS_BUFFER_CACHE_GC_CALLOUTS_MAX_SIZE; i++) { if (fs_callouts[i].callout == NULL) { fs_callouts[i].callout = callout; fs_callouts[i].context = context; lck_mtx_unlock(&buf_gc_callout); return 0; } } lck_mtx_unlock(&buf_gc_callout); return ENOMEM; } int fs_buffer_cache_gc_unregister(void (* callout)(int, void *), void *context) { lck_mtx_lock(&buf_gc_callout); for (int i = 0; i < FS_BUFFER_CACHE_GC_CALLOUTS_MAX_SIZE; i++) { if (fs_callouts[i].callout == callout && fs_callouts[i].context == context) { fs_callouts[i].callout = NULL; fs_callouts[i].context = NULL; } } lck_mtx_unlock(&buf_gc_callout); return 0; } static void fs_buffer_cache_gc_dispatch_callouts(int all) { lck_mtx_lock(&buf_gc_callout); for (int i = 0; i < FS_BUFFER_CACHE_GC_CALLOUTS_MAX_SIZE; i++) { if (fs_callouts[i].callout != NULL) { fs_callouts[i].callout(all, fs_callouts[i].context); } } lck_mtx_unlock(&buf_gc_callout); } static boolean_t buffer_cache_gc(int all) { buf_t bp; boolean_t did_large_zfree = FALSE; boolean_t need_wakeup = FALSE; int now = buf_timestamp(); uint32_t found = 0; struct bqueues privq; int thresh_hold = BUF_STALE_THRESHHOLD; if (all) { thresh_hold = 0; } /* * We only care about metadata (incore storage comes from zalloc()). * Unless "all" is set (used to evict meta data buffers in preparation * for deep sleep), we only evict up to BUF_MAX_GC_BATCH_SIZE buffers * that have not been accessed in the last BUF_STALE_THRESHOLD seconds. * BUF_MAX_GC_BATCH_SIZE controls both the hold time of the global lock * "buf_mtx" and the length of time we spend compute bound in the GC * thread which calls this function */ lck_mtx_lock(&buf_mtx); do { found = 0; TAILQ_INIT(&privq); need_wakeup = FALSE; while (((bp = TAILQ_FIRST(&bufqueues[BQ_META]))) && (now > bp->b_timestamp) && (now - bp->b_timestamp > thresh_hold) && (found < BUF_MAX_GC_BATCH_SIZE)) { /* Remove from free list */ bremfree_locked(bp); found++; #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 12; #endif /* If dirty, move to laundry queue and remember to do wakeup */ if (ISSET(bp->b_flags, B_DELWRI)) { SET(bp->b_lflags, BL_WANTDEALLOC); bmovelaundry(bp); need_wakeup = TRUE; continue; } /* * Mark busy and put on private list. We could technically get * away without setting BL_BUSY here. */ SET(bp->b_lflags, BL_BUSY); buf_busycount++; /* * Remove from hash and dissociate from vp. */ bremhash(bp); if (bp->b_vp) { brelvp_locked(bp); } TAILQ_INSERT_TAIL(&privq, bp, b_freelist); } if (found == 0) { break; } /* Drop lock for batch processing */ lck_mtx_unlock(&buf_mtx); /* Wakeup and yield for laundry if need be */ if (need_wakeup) { wakeup(&bufqueues[BQ_LAUNDRY]); (void)thread_block(THREAD_CONTINUE_NULL); } /* Clean up every buffer on private list */ TAILQ_FOREACH(bp, &privq, b_freelist) { /* Take note if we've definitely freed at least a page to a zone */ if ((ISSET(bp->b_flags, B_ZALLOC)) && (buf_size(bp) >= PAGE_SIZE)) { did_large_zfree = TRUE; } trace(TR_BRELSE, pack(bp->b_vp, bp->b_bufsize), bp->b_lblkno); /* Free Storage */ buf_free_meta_store(bp); /* Release credentials */ buf_release_credentials(bp); /* Prepare for moving to empty queue */ CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA)); bp->b_whichq = BQ_EMPTY; BLISTNONE(bp); } lck_mtx_lock(&buf_mtx); /* Back under lock, move them all to invalid hash and clear busy */ TAILQ_FOREACH(bp, &privq, b_freelist) { binshash(bp, &invalhash); CLR(bp->b_lflags, BL_BUSY); buf_busycount--; #ifdef JOE_DEBUG if (bp->b_owner != current_thread()) { panic("Buffer stolen from buffer_cache_gc()"); } bp->b_owner = current_thread(); bp->b_tag = 13; #endif } /* And do a big bulk move to the empty queue */ TAILQ_CONCAT(&bufqueues[BQ_EMPTY], &privq, b_freelist); } while (all && (found == BUF_MAX_GC_BATCH_SIZE)); lck_mtx_unlock(&buf_mtx); fs_buffer_cache_gc_dispatch_callouts(all); return did_large_zfree; } /* * disabled for now */ #if FLUSH_QUEUES #define NFLUSH 32 static int bp_cmp(void *a, void *b) { buf_t *bp_a = *(buf_t **)a, *bp_b = *(buf_t **)b; daddr64_t res; // don't have to worry about negative block // numbers so this is ok to do. // res = (bp_a->b_blkno - bp_b->b_blkno); return (int)res; } int bflushq(int whichq, mount_t mp) { buf_t bp, next; int i, buf_count; int total_writes = 0; static buf_t flush_table[NFLUSH]; if (whichq < 0 || whichq >= BQUEUES) { return 0; } restart: lck_mtx_lock(&buf_mtx); bp = TAILQ_FIRST(&bufqueues[whichq]); for (buf_count = 0; bp; bp = next) { next = bp->b_freelist.tqe_next; if (bp->b_vp == NULL || bp->b_vp->v_mount != mp) { continue; } if (ISSET(bp->b_flags, B_DELWRI) && !ISSET(bp->b_lflags, BL_BUSY)) { bremfree_locked(bp); #ifdef JOE_DEBUG bp->b_owner = current_thread(); bp->b_tag = 7; #endif SET(bp->b_lflags, BL_BUSY); buf_busycount++; flush_table[buf_count] = bp; buf_count++; total_writes++; if (buf_count >= NFLUSH) { lck_mtx_unlock(&buf_mtx); qsort(flush_table, buf_count, sizeof(struct buf *), bp_cmp); for (i = 0; i < buf_count; i++) { buf_bawrite(flush_table[i]); } goto restart; } } } lck_mtx_unlock(&buf_mtx); if (buf_count > 0) { qsort(flush_table, buf_count, sizeof(struct buf *), bp_cmp); for (i = 0; i < buf_count; i++) { buf_bawrite(flush_table[i]); } } return total_writes; } #endif