/* * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Copyright (c) 1998 Apple Computer, Inc. All rights reserved. * * File: bsd/kern/kern_symfile.c * * HISTORY */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define HIBERNATE_MIN_PHYSICAL_LBA_512 (34) #define HIBERNATE_MIN_PHYSICAL_LBA_4096 (6) #define HIBERNATE_MIN_FILE_SIZE (1024*1024) /* This function is called from kern_sysctl in the current process context; * it is exported with the System6.0.exports, but this appears to be a legacy * export, as there are no internal consumers. */ int get_kernel_symfile(__unused proc_t p, __unused char const **symfile); int get_kernel_symfile(__unused proc_t p, __unused char const **symfile) { return KERN_FAILURE; } struct kern_direct_file_io_ref_t { vfs_context_t ctx; struct vnode * vp; char * name; size_t namesize; dev_t device; uint32_t blksize; off_t filelength; char cf; char pinned; char frozen; char wbcranged; }; static int file_ioctl(void * p1, void * p2, u_long theIoctl, caddr_t result) { dev_t device = *(dev_t*) p1; return (*bdevsw[major(device)].d_ioctl) (device, theIoctl, result, S_IFBLK, p2); } static int device_ioctl(void * p1, __unused void * p2, u_long theIoctl, caddr_t result) { return VNOP_IOCTL(p1, theIoctl, result, 0, p2); } static int kern_ioctl_file_extents(struct kern_direct_file_io_ref_t * ref, u_long theIoctl, off_t offset, off_t end) { int error = 0; int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); void * p1; void * p2; uint64_t fileblk = 0; size_t filechunk = 0; dk_extent_t extent; dk_unmap_t unmap; _dk_cs_pin_t pin; bzero(&extent, sizeof(dk_extent_t)); bzero(&unmap, sizeof(dk_unmap_t)); bzero(&pin, sizeof(pin)); if (ref->vp->v_type == VREG) { p1 = &ref->device; p2 = kernproc; do_ioctl = &file_ioctl; } else { /* Partition. */ p1 = ref->vp; p2 = ref->ctx; do_ioctl = &device_ioctl; } if (_DKIOCCSPINEXTENT == theIoctl) { /* Tell CS the image size, so it knows whether to place the subsequent pins SSD/HDD */ pin.cp_extent.length = end; pin.cp_flags = _DKIOCCSHIBERNATEIMGSIZE; (void) do_ioctl(p1, p2, _DKIOCCSPINEXTENT, (caddr_t)&pin); } else if (_DKIOCCSUNPINEXTENT == theIoctl) { /* Tell CS hibernation is done, so it can stop blocking overlapping writes */ pin.cp_flags = _DKIOCCSPINDISCARDDENYLIST; (void) do_ioctl(p1, p2, _DKIOCCSUNPINEXTENT, (caddr_t)&pin); } for (; offset < end; offset += filechunk) { if (ref->vp->v_type == VREG) { daddr64_t blkno; filechunk = 1 * 1024 * 1024 * 1024; if (filechunk > (size_t)(end - offset)) { filechunk = (size_t)(end - offset); } error = VNOP_BLOCKMAP(ref->vp, offset, filechunk, &blkno, &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) { break; } if (-1LL == blkno) { continue; } fileblk = blkno * ref->blksize; } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { fileblk = offset; filechunk = (unsigned long)((ref->filelength > ULONG_MAX) ? ULONG_MAX: ref->filelength); } if (DKIOCUNMAP == theIoctl) { extent.offset = fileblk; extent.length = filechunk; unmap.extents = &extent; unmap.extentsCount = 1; error = do_ioctl(p1, p2, theIoctl, (caddr_t)&unmap); // printf("DKIOCUNMAP(%d) 0x%qx, 0x%qx\n", error, extent.offset, extent.length); } else if (_DKIOCCSPINEXTENT == theIoctl) { pin.cp_extent.offset = fileblk; pin.cp_extent.length = filechunk; pin.cp_flags = _DKIOCCSPINFORHIBERNATION; error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin); if (error && (ENOTTY != error)) { printf("_DKIOCCSPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length); } } else if (_DKIOCCSUNPINEXTENT == theIoctl) { pin.cp_extent.offset = fileblk; pin.cp_extent.length = filechunk; pin.cp_flags = _DKIOCCSPINFORHIBERNATION; error = do_ioctl(p1, p2, theIoctl, (caddr_t)&pin); if (error && (ENOTTY != error)) { printf("_DKIOCCSUNPINEXTENT(%d) 0x%qx, 0x%qx\n", error, pin.cp_extent.offset, pin.cp_extent.length); } } else { error = EINVAL; } if (error) { break; } } return error; } extern uint32_t freespace_mb(vnode_t vp); struct kern_direct_file_io_ref_t * kern_open_file_for_direct_io(const char * name, uint32_t iflags, kern_get_file_extents_callback_t callback, void * callback_ref, off_t set_file_size, off_t fs_free_size, off_t write_file_offset, void * write_file_addr, size_t write_file_len, dev_t * partition_device_result, dev_t * image_device_result, uint64_t * partitionbase_result, uint64_t * maxiocount_result, uint32_t * oflags) { struct kern_direct_file_io_ref_t * ref; proc_t p; struct vnode_attr va; dk_apfs_wbc_range_t wbc_range; int error; off_t f_offset; uint64_t fileblk = 0; size_t filechunk = 0; uint64_t physoffset, minoffset; dev_t device; dev_t target = 0; int isssd = 0; uint32_t flags = 0; uint32_t blksize; off_t maxiocount, count, segcount, wbctotal; boolean_t locked = FALSE; int fmode; mode_t cmode; struct nameidata nd; u_int32_t ndflags; off_t mpFree; wbc_range.count = 0; int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); do_ioctl = NULL; void * p1 = NULL; void * p2 = NULL; error = EFAULT; ref = kalloc_type(struct kern_direct_file_io_ref_t, Z_WAITOK | Z_ZERO | Z_NOFAIL); p = kernproc; ref->ctx = vfs_context_kernel(); ref->namesize = strlen(name) + 1; ref->name = kalloc_data(ref->namesize, Z_WAITOK | Z_NOFAIL); strlcpy(ref->name, name, ref->namesize); fmode = (kIOPolledFileCreate & iflags) ? (O_CREAT | FWRITE) : FWRITE; cmode = S_IRUSR | S_IWUSR; ndflags = NOFOLLOW; NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(ref->name), ref->ctx); VATTR_INIT(&va); VATTR_SET(&va, va_mode, cmode); VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED); VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D); if ((error = vn_open_auth(&nd, &fmode, &va, NULLVP))) { kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error); goto out; } ref->vp = nd.ni_vp; if (ref->vp->v_type == VREG) { vnode_lock_spin(ref->vp); SET(ref->vp->v_flag, VSWAP); vnode_unlock(ref->vp); } if (write_file_addr && write_file_len) { if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) { kprintf("kern_write_file() failed with error: %d\n", error); goto out; } } VATTR_INIT(&va); VATTR_WANTED(&va, va_rdev); VATTR_WANTED(&va, va_fsid); VATTR_WANTED(&va, va_devid); VATTR_WANTED(&va, va_data_size); VATTR_WANTED(&va, va_data_alloc); VATTR_WANTED(&va, va_nlink); error = EFAULT; if (vnode_getattr(ref->vp, &va, ref->ctx)) { goto out; } wbctotal = 0; mpFree = freespace_mb(ref->vp); mpFree <<= 20; kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n", ref->name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size); if (ref->vp->v_type == VREG) { /* Don't dump files with links. */ if (va.va_nlink != 1) { goto out; } /* Don't dump on fs without backing device. */ if (!VATTR_IS_SUPPORTED(&va, va_devid)) { kprintf("kern_direct_file(%s): Not backed by block device.\n", ref->name); error = ENODEV; goto out; } device = va.va_devid; ref->filelength = va.va_data_size; p1 = &device; p2 = p; do_ioctl = &file_ioctl; if (kIOPolledFileHibernate & iflags) { error = do_ioctl(p1, p2, DKIOCAPFSGETWBCRANGE, (caddr_t) &wbc_range); ref->wbcranged = (error == 0); } if (ref->wbcranged) { uint32_t idx; assert(wbc_range.count <= (sizeof(wbc_range.extents) / sizeof(wbc_range.extents[0]))); for (idx = 0; idx < wbc_range.count; idx++) { wbctotal += wbc_range.extents[idx].length; } kprintf("kern_direct_file(%s): wbc %qd\n", ref->name, wbctotal); if (wbctotal) { target = wbc_range.dev; } } if (set_file_size) { if (wbctotal) { if (wbctotal >= set_file_size) { set_file_size = HIBERNATE_MIN_FILE_SIZE; } else { set_file_size -= wbctotal; if (set_file_size < HIBERNATE_MIN_FILE_SIZE) { set_file_size = HIBERNATE_MIN_FILE_SIZE; } } } if (fs_free_size) { mpFree += va.va_data_alloc; if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size)) { error = ENOSPC; goto out; } } error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx); if (error) { goto out; } ref->filelength = set_file_size; } } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { /* Partition. */ device = va.va_rdev; p1 = ref->vp; p2 = ref->ctx; do_ioctl = &device_ioctl; } else { /* Don't dump to non-regular files. */ error = EFAULT; goto out; } ref->device = device; // probe for CF dk_corestorage_info_t cs_info; memset(&cs_info, 0, sizeof(dk_corestorage_info_t)); error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info); ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES); // get block size error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize); if (error) { goto out; } if (ref->blksize == 4096) { minoffset = HIBERNATE_MIN_PHYSICAL_LBA_4096 * ref->blksize; } else { minoffset = HIBERNATE_MIN_PHYSICAL_LBA_512 * ref->blksize; } if (ref->vp->v_type != VREG) { error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk); if (error) { goto out; } ref->filelength = fileblk * ref->blksize; } // pin logical extents, CS version error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength); if (error && (ENOTTY != error)) { goto out; } ref->pinned = (error == 0); // pin logical extents, apfs version error = VNOP_IOCTL(ref->vp, FSCTL_FREEZE_EXTENTS, NULL, 0, ref->ctx); if (error && (ENOTTY != error)) { goto out; } ref->frozen = (error == 0); // generate the block list error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL); if (error) { goto out; } locked = TRUE; f_offset = 0; for (; f_offset < ref->filelength; f_offset += filechunk) { if (ref->vp->v_type == VREG) { filechunk = 1 * 1024 * 1024 * 1024; daddr64_t blkno; error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno, &filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL); if (error) { goto out; } if (-1LL == blkno) { continue; } fileblk = blkno * ref->blksize; } else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { fileblk = f_offset; filechunk = f_offset ? 0 : (unsigned long)ref->filelength; } physoffset = 0; while (physoffset < filechunk) { dk_physical_extent_t getphysreq; bzero(&getphysreq, sizeof(getphysreq)); getphysreq.offset = fileblk + physoffset; getphysreq.length = (filechunk - physoffset); error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq); if (error) { goto out; } if (!target) { target = getphysreq.dev; } else if (target != getphysreq.dev) { error = ENOTSUP; goto out; } assert(getphysreq.offset >= minoffset); #if HIBFRAGMENT uint64_t rev; for (rev = 4096; rev <= getphysreq.length; rev += 4096) { callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096); } #else callback(callback_ref, getphysreq.offset, getphysreq.length); #endif physoffset += getphysreq.length; } } if (ref->wbcranged) { uint32_t idx; for (idx = 0; idx < wbc_range.count; idx++) { assert(wbc_range.extents[idx].offset >= minoffset); callback(callback_ref, wbc_range.extents[idx].offset, wbc_range.extents[idx].length); } } callback(callback_ref, 0ULL, 0ULL); if (ref->vp->v_type == VREG) { p1 = ⌖ } else { p1 = ⌖ p2 = p; do_ioctl = &file_ioctl; } // get partition base if (partitionbase_result) { error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result); if (error) { goto out; } } // get block size & constraints error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize); if (error) { goto out; } maxiocount = 1 * 1024 * 1024 * 1024; error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count); if (error) { count = 0; } count *= blksize; if (count && (count < maxiocount)) { maxiocount = count; } error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count); if (error) { count = 0; } count *= blksize; if (count && (count < maxiocount)) { maxiocount = count; } error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count); if (error) { count = 0; } if (count && (count < maxiocount)) { maxiocount = count; } error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count); if (error) { count = 0; } if (count && (count < maxiocount)) { maxiocount = count; } error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count); if (!error) { error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount); } if (error) { count = segcount = 0; } count *= segcount; if (count && (count < maxiocount)) { maxiocount = count; } error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count); if (!error) { error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount); } if (error) { count = segcount = 0; } count *= segcount; if (count && (count < maxiocount)) { maxiocount = count; } kprintf("max io 0x%qx bytes\n", maxiocount); if (maxiocount_result) { *maxiocount_result = maxiocount; } error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd); if (!error && isssd) { flags |= kIOPolledFileSSD; } if (partition_device_result) { *partition_device_result = device; } if (image_device_result) { *image_device_result = target; } if (oflags) { *oflags = flags; } if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR)) { vnode_close(ref->vp, FWRITE, ref->ctx); ref->vp = NULLVP; ref->ctx = NULL; } out: printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error); if (error && locked) { p1 = &device; if (do_ioctl) { (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL); } } if (error && ref) { if (ref->vp) { (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0); if (ref->frozen) { (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx); } if (ref->wbcranged) { if (do_ioctl) { (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL); } } vnode_close(ref->vp, FWRITE, ref->ctx); ref->vp = NULLVP; } ref->ctx = NULL; if (ref->name) { kfree_data(ref->name, ref->namesize); ref->name = NULL; } kfree_type(struct kern_direct_file_io_ref_t, ref); ref = NULL; } return ref; } int kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag) { assert(len <= INT32_MAX); return vn_rdwr(UIO_WRITE, ref->vp, addr, (int)len, offset, UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT, vfs_context_ucred(ref->ctx), (int *) 0, vfs_context_proc(ref->ctx)); } int kern_read_file(struct kern_direct_file_io_ref_t * ref, off_t offset, void * addr, size_t len, int ioflag) { assert(len <= INT32_MAX); return vn_rdwr(UIO_READ, ref->vp, addr, (int)len, offset, UIO_SYSSPACE, ioflag | IO_SYNC | IO_NODELOCKED | IO_UNIT, vfs_context_ucred(ref->ctx), (int *) 0, vfs_context_proc(ref->ctx)); } struct mount * kern_file_mount(struct kern_direct_file_io_ref_t * ref) { return ref->vp->v_mount; } void kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref, off_t write_offset, void * addr, size_t write_length, off_t discard_offset, off_t discard_end, bool unlink) { int error; printf("kern_close_file_for_direct_io(%p)\n", ref); if (!ref) { return; } if (ref->vp) { int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result); void * p1; void * p2; discard_offset = ((discard_offset + ref->blksize - 1) & ~(((off_t) ref->blksize) - 1)); discard_end = ((discard_end) & ~(((off_t) ref->blksize) - 1)); if (ref->vp->v_type == VREG) { p1 = &ref->device; p2 = kernproc; do_ioctl = &file_ioctl; } else { /* Partition. */ p1 = ref->vp; p2 = ref->ctx; do_ioctl = &device_ioctl; } (void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL); //XXX If unmapping extents then don't also need to unpin; except ... //XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and //XXX unmap is subset, so save extra walk over file extents (and the risk //XXX that CF drain starts) vs leaving partial units pinned to SSD //XXX (until whatever was sharing also unmaps). Err on cleaning up fully. boolean_t will_unmap = (!ref->pinned || ref->cf) && (discard_end > discard_offset); boolean_t will_unpin = (ref->pinned && ref->cf /* && !will_unmap */); (void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0); if (will_unmap) { (void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end); } if (ref->frozen) { (void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx); } if (ref->wbcranged) { (void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL); } if (addr && write_length) { (void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION); } error = vnode_close(ref->vp, FWRITE, ref->ctx); ref->vp = NULLVP; kprintf("vnode_close(%d)\n", error); if (unlink) { int unlink1(vfs_context_t, vnode_t, user_addr_t, enum uio_seg, int); error = unlink1(ref->ctx, NULLVP, CAST_USER_ADDR_T(ref->name), UIO_SYSSPACE, 0); kprintf("%s: unlink1(%d)\n", __func__, error); } } ref->ctx = NULL; kfree_data(ref->name, ref->namesize); ref->name = NULL; kfree_type(struct kern_direct_file_io_ref_t, ref); }