3257 lines
113 KiB
C
3257 lines
113 KiB
C
/*
|
||
* Copyright (c) 2006-2018 Apple Inc. All rights reserved.
|
||
*
|
||
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
|
||
*
|
||
* This file contains Original Code and/or Modifications of Original Code
|
||
* as defined in and that are subject to the Apple Public Source License
|
||
* Version 2.0 (the 'License'). You may not use this file except in
|
||
* compliance with the License. The rights granted to you under the License
|
||
* may not be used to create, or enable the creation or redistribution of,
|
||
* unlawful or unlicensed copies of an Apple operating system, or to
|
||
* circumvent, violate, or enable the circumvention or violation of, any
|
||
* terms of an Apple operating system software license agreement.
|
||
*
|
||
* Please obtain a copy of the License at
|
||
* http://www.opensource.apple.com/apsl/ and read it before using this file.
|
||
*
|
||
* The Original Code and all software distributed under the License are
|
||
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
|
||
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
|
||
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
|
||
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
|
||
* Please see the License for the specific language governing rights and
|
||
* limitations under the License.
|
||
*
|
||
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
|
||
*
|
||
*/
|
||
|
||
#include <kern/sched_prim.h>
|
||
#include <kern/kalloc.h>
|
||
#include <kern/assert.h>
|
||
#include <kern/debug.h>
|
||
#include <kern/locks.h>
|
||
#include <kern/task.h>
|
||
#include <kern/thread.h>
|
||
#include <kern/host.h>
|
||
#include <kern/policy_internal.h>
|
||
#include <kern/thread_call.h>
|
||
#include <kern/thread_group.h>
|
||
|
||
#include <libkern/libkern.h>
|
||
#include <mach/coalition.h>
|
||
#include <mach/mach_time.h>
|
||
#include <mach/task.h>
|
||
#include <mach/host_priv.h>
|
||
#include <mach/mach_host.h>
|
||
#include <os/log.h>
|
||
#include <pexpert/pexpert.h>
|
||
#include <sys/coalition.h>
|
||
#include <sys/kern_event.h>
|
||
#include <sys/kdebug.h>
|
||
#include <sys/kdebug_kernel.h>
|
||
#include <sys/proc.h>
|
||
#include <sys/proc_info.h>
|
||
#include <sys/reason.h>
|
||
#include <sys/signal.h>
|
||
#include <sys/signalvar.h>
|
||
#include <sys/sysctl.h>
|
||
#include <sys/sysproto.h>
|
||
#include <sys/wait.h>
|
||
#include <sys/tree.h>
|
||
#include <sys/priv.h>
|
||
#include <vm/vm_pageout.h>
|
||
#include <vm/vm_protos.h>
|
||
#include <mach/machine/sdt.h>
|
||
#include <libkern/coreanalytics/coreanalytics.h>
|
||
#include <libkern/section_keywords.h>
|
||
#include <stdatomic.h>
|
||
|
||
#include <IOKit/IOBSD.h>
|
||
|
||
#if CONFIG_FREEZE
|
||
#include <vm/vm_map.h>
|
||
#endif /* CONFIG_FREEZE */
|
||
|
||
#include <kern/kern_memorystatus_internal.h>
|
||
#include <sys/kern_memorystatus.h>
|
||
#include <sys/kern_memorystatus_freeze.h>
|
||
#include <sys/kern_memorystatus_notify.h>
|
||
|
||
#if CONFIG_JETSAM
|
||
|
||
extern unsigned int memorystatus_available_pages;
|
||
extern unsigned int memorystatus_available_pages_pressure;
|
||
extern unsigned int memorystatus_available_pages_critical;
|
||
extern unsigned int memorystatus_available_pages_critical_base;
|
||
extern unsigned int memorystatus_available_pages_critical_idle_offset;
|
||
|
||
#else /* CONFIG_JETSAM */
|
||
|
||
extern uint64_t memorystatus_available_pages;
|
||
extern uint64_t memorystatus_available_pages_pressure;
|
||
extern uint64_t memorystatus_available_pages_critical;
|
||
|
||
#endif /* CONFIG_JETSAM */
|
||
|
||
unsigned int memorystatus_frozen_count = 0;
|
||
unsigned int memorystatus_frozen_count_webcontent = 0;
|
||
unsigned int memorystatus_frozen_count_xpc_service = 0;
|
||
unsigned int memorystatus_suspended_count = 0;
|
||
|
||
#if CONFIG_FREEZE
|
||
|
||
static LCK_GRP_DECLARE(freezer_lck_grp, "freezer");
|
||
static LCK_MTX_DECLARE(freezer_mutex, &freezer_lck_grp);
|
||
|
||
/* Thresholds */
|
||
unsigned int memorystatus_freeze_threshold = 0;
|
||
unsigned int memorystatus_freeze_pages_min = 0;
|
||
unsigned int memorystatus_freeze_pages_max = 0;
|
||
unsigned int memorystatus_freeze_suspended_threshold = FREEZE_SUSPENDED_THRESHOLD_DEFAULT;
|
||
unsigned int memorystatus_freeze_daily_mb_max = FREEZE_DAILY_MB_MAX_DEFAULT;
|
||
uint64_t memorystatus_freeze_budget_pages_remaining = 0; /* Remaining # of pages that can be frozen to disk */
|
||
uint64_t memorystatus_freeze_budget_multiplier = 100; /* Multiplies the daily budget by 100/multiplier */
|
||
boolean_t memorystatus_freeze_degradation = FALSE; /* Protected by the freezer mutex. Signals we are in a degraded freeze mode. */
|
||
unsigned int memorystatus_freeze_max_candidate_band = FREEZE_MAX_CANDIDATE_BAND;
|
||
|
||
unsigned int memorystatus_max_frozen_demotions_daily = 0;
|
||
unsigned int memorystatus_thaw_count_demotion_threshold = 0;
|
||
unsigned int memorystatus_min_thaw_refreeze_threshold;
|
||
|
||
#if XNU_TARGET_OS_WATCH
|
||
#define FREEZE_DYNAMIC_THREAD_DELAY_ENABLED_DEFAULT true
|
||
#else
|
||
#define FREEZE_DYNAMIC_THREAD_DELAY_ENABLED_DEFAULT false
|
||
#endif
|
||
boolean_t memorystatus_freeze_dynamic_thread_delay_enabled = FREEZE_DYNAMIC_THREAD_DELAY_ENABLED_DEFAULT;
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_dynamic_thread_delay_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_freeze_dynamic_thread_delay_enabled, 0, "");
|
||
|
||
#define FREEZE_APPS_IDLE_DELAY_MULTIPLIER_FAST 1
|
||
#define FREEZE_APPS_IDLE_DELAY_MULTIPLIER_SLOW 30
|
||
#define FREEZE_APPS_IDLE_DELAY_MULTIPLIER_DEFAULT FREEZE_APPS_IDLE_DELAY_MULTIPLIER_FAST
|
||
unsigned int memorystatus_freeze_apps_idle_delay_multiplier = FREEZE_APPS_IDLE_DELAY_MULTIPLIER_DEFAULT;
|
||
|
||
#if (XNU_TARGET_OS_IOS && || XNU_TARGET_OS_WATCH
|
||
#define FREEZE_ENABLED_DEFAULT true
|
||
#else
|
||
#define FREEZE_ENABLED_DEFAULT false
|
||
#endif
|
||
TUNABLE_WRITEABLE(bool, memorystatus_freeze_enabled, "freeze_enabled", FREEZE_ENABLED_DEFAULT);
|
||
|
||
int memorystatus_freeze_wakeup = 0;
|
||
int memorystatus_freeze_jetsam_band = 0; /* the jetsam band which will contain P_MEMSTAT_FROZEN processes */
|
||
|
||
#define MAX_XPC_SERVICE_PIDS 10 /* Max. # of XPC services per coalition we'll consider freezing. */
|
||
|
||
#ifdef XNU_KERNEL_PRIVATE
|
||
|
||
unsigned int memorystatus_frozen_processes_max = 0;
|
||
unsigned int memorystatus_frozen_shared_mb = 0;
|
||
unsigned int memorystatus_frozen_shared_mb_max = 0;
|
||
unsigned int memorystatus_freeze_shared_mb_per_process_max = 0; /* Max. MB allowed per process to be freezer-eligible. */
|
||
#if XNU_TARGET_OS_WATCH
|
||
unsigned int memorystatus_freeze_private_shared_pages_ratio = 1; /* Ratio of private:shared pages for a process to be freezer-eligible. */
|
||
#else
|
||
unsigned int memorystatus_freeze_private_shared_pages_ratio = 2; /* Ratio of private:shared pages for a process to be freezer-eligible. */
|
||
#endif
|
||
unsigned int memorystatus_thaw_count = 0; /* # of thaws in the current freezer interval */
|
||
uint64_t memorystatus_thaw_count_since_boot = 0; /* The number of thaws since boot */
|
||
unsigned int memorystatus_refreeze_eligible_count = 0; /* # of processes currently thawed i.e. have state on disk & in-memory */
|
||
|
||
struct memorystatus_freezer_stats_t memorystatus_freezer_stats = {0};
|
||
|
||
#endif /* XNU_KERNEL_PRIVATE */
|
||
|
||
static inline boolean_t memorystatus_can_freeze_processes(void);
|
||
static boolean_t memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low);
|
||
static void memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused);
|
||
static uint32_t memorystatus_freeze_calculate_new_budget(
|
||
unsigned int time_since_last_interval_expired_sec,
|
||
unsigned int burst_multiple,
|
||
unsigned int interval_duration_min,
|
||
uint32_t rollover);
|
||
static void memorystatus_freeze_start_normal_throttle_interval(uint32_t new_budget, mach_timespec_t start_ts);
|
||
|
||
static void memorystatus_set_freeze_is_enabled(bool enabled);
|
||
static void memorystatus_disable_freeze(void);
|
||
static bool kill_all_frozen_processes(uint64_t max_band, bool suspended_only, os_reason_t jetsam_reason, uint64_t *memory_reclaimed_out);
|
||
|
||
/* Stats */
|
||
static uint64_t memorystatus_freeze_pageouts = 0;
|
||
|
||
/* Throttling */
|
||
#define DEGRADED_WINDOW_MINS (30)
|
||
#define NORMAL_WINDOW_MINS (24 * 60)
|
||
|
||
/* Protected by the freezer_mutex */
|
||
static throttle_interval_t throttle_intervals[] = {
|
||
{ DEGRADED_WINDOW_MINS, 1, 0, 0, { 0, 0 }},
|
||
{ NORMAL_WINDOW_MINS, 1, 0, 0, { 0, 0 }},
|
||
};
|
||
throttle_interval_t *degraded_throttle_window = &throttle_intervals[0];
|
||
throttle_interval_t *normal_throttle_window = &throttle_intervals[1];
|
||
uint32_t memorystatus_freeze_current_interval = 0;
|
||
static thread_call_t freeze_interval_reset_thread_call;
|
||
static uint32_t memorystatus_freeze_calculate_new_budget(
|
||
unsigned int time_since_last_interval_expired_sec,
|
||
unsigned int burst_multiple,
|
||
unsigned int interval_duration_min,
|
||
uint32_t rollover);
|
||
|
||
struct memorystatus_freezer_candidate_list memorystatus_global_freeze_list = {NULL, 0};
|
||
struct memorystatus_freezer_candidate_list memorystatus_global_demote_list = {NULL, 0};
|
||
/*
|
||
* When enabled, freeze candidates are chosen from the memorystatus_global_freeze_list
|
||
* in order (as opposed to using the older LRU approach).
|
||
*/
|
||
#if XNU_TARGET_OS_WATCH
|
||
#define FREEZER_USE_ORDERED_LIST_DEFAULT 1
|
||
#else
|
||
#define FREEZER_USE_ORDERED_LIST_DEFAULT 0
|
||
#endif
|
||
int memorystatus_freezer_use_ordered_list = FREEZER_USE_ORDERED_LIST_DEFAULT;
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freezer_use_ordered_list, &memorystatus_freezer_use_ordered_list, 0, 1, "");
|
||
/*
|
||
* When enabled, demotion candidates are chosen from memorystatus_global_demotion_list
|
||
*/
|
||
int memorystatus_freezer_use_demotion_list = 0;
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freezer_use_demotion_list, &memorystatus_freezer_use_demotion_list, 0, 1, "");
|
||
|
||
extern uint64_t vm_swap_get_free_space(void);
|
||
extern boolean_t vm_swap_max_budget(uint64_t *);
|
||
|
||
static void memorystatus_freeze_update_throttle(uint64_t *budget_pages_allowed);
|
||
static void memorystatus_demote_frozen_processes(bool urgent_mode);
|
||
|
||
static void memorystatus_freeze_handle_error(proc_t p, const freezer_error_code_t freezer_error_code, bool was_refreeze, pid_t pid, const coalition_t coalition, const char* log_prefix);
|
||
static void memorystatus_freeze_out_of_slots(void);
|
||
uint64_t memorystatus_freezer_thread_next_run_ts = 0;
|
||
|
||
/* Sysctls needed for aggd stats */
|
||
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_frozen_count, 0, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_count_webcontent, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_frozen_count_webcontent, 0, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_count_xpc_service, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_frozen_count_xpc_service, 0, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_thaw_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_thaw_count, 0, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_thaw_count_since_boot, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_thaw_count_since_boot, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_pageouts, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freeze_pageouts, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_interval, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freeze_current_interval, 0, "");
|
||
|
||
/*
|
||
* Force a new interval with the given budget (no rollover).
|
||
*/
|
||
static void
|
||
memorystatus_freeze_force_new_interval(uint64_t new_budget)
|
||
{
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
mach_timespec_t now_ts;
|
||
clock_sec_t sec;
|
||
clock_nsec_t nsec;
|
||
|
||
clock_get_system_nanotime(&sec, &nsec);
|
||
now_ts.tv_sec = (unsigned int)(MIN(sec, UINT32_MAX));
|
||
now_ts.tv_nsec = nsec;
|
||
memorystatus_freeze_start_normal_throttle_interval((uint32_t) MIN(new_budget, UINT32_MAX), now_ts);
|
||
/* Don't carry over any excess pageouts since we're forcing a new budget */
|
||
normal_throttle_window->pageouts = 0;
|
||
memorystatus_freeze_budget_pages_remaining = normal_throttle_window->max_pageouts;
|
||
}
|
||
#if DEVELOPMENT || DEBUG
|
||
static int sysctl_memorystatus_freeze_budget_pages_remaining SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2, oidp)
|
||
int error, changed;
|
||
uint64_t new_budget = memorystatus_freeze_budget_pages_remaining;
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
|
||
error = sysctl_io_number(req, memorystatus_freeze_budget_pages_remaining, sizeof(uint64_t), &new_budget, &changed);
|
||
if (changed) {
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
return ENOTSUP;
|
||
}
|
||
memorystatus_freeze_force_new_interval(new_budget);
|
||
}
|
||
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
return error;
|
||
}
|
||
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freeze_budget_pages_remaining, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0, &sysctl_memorystatus_freeze_budget_pages_remaining, "Q", "");
|
||
#else /* DEVELOPMENT || DEBUG */
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freeze_budget_pages_remaining, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freeze_budget_pages_remaining, "");
|
||
#endif /* DEVELOPMENT || DEBUG */
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_excess_shared_memory_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_excess_shared_memory_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_low_private_shared_ratio_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_low_private_shared_ratio_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_no_compressor_space_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_no_compressor_space_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_no_swap_space_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_no_swap_space_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_below_min_pages_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_below_min_pages_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_low_probability_of_use_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_low_probability_of_use_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_elevated_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_elevated_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_error_other_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_error_other_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_process_considered_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_process_considered_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_below_threshold_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_below_threshold_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_skipped_full_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_skipped_full_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_skipped_shared_mb_high_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_skipped_shared_mb_high_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_shared_pages_skipped, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_shared_pages_skipped, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_bytes_refrozen, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_bytes_refrozen, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_refreeze_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_refreeze_count, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_freeze_pid_mismatches, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_freeze_pid_mismatches, "");
|
||
SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_freezer_demote_pid_mismatches, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freezer_stats.mfs_demote_pid_mismatches, "");
|
||
|
||
static_assert(_kMemorystatusFreezeSkipReasonMax <= UINT8_MAX);
|
||
|
||
/*
|
||
* Calculates the hit rate for the freezer.
|
||
* The hit rate is defined as the percentage of procs that are currently in the
|
||
* freezer which we have thawed.
|
||
* A low hit rate means we're freezing bad candidates since they're not re-used.
|
||
*/
|
||
static int
|
||
calculate_thaw_percentage(uint64_t frozen_count, uint64_t thaw_count)
|
||
{
|
||
int thaw_percentage = 100;
|
||
|
||
if (frozen_count > 0) {
|
||
if (thaw_count > frozen_count) {
|
||
/*
|
||
* Both counts are using relaxed atomics & could be out of sync
|
||
* causing us to see thaw_percentage > 100.
|
||
*/
|
||
thaw_percentage = 100;
|
||
} else {
|
||
thaw_percentage = (int)(100 * thaw_count / frozen_count);
|
||
}
|
||
}
|
||
return thaw_percentage;
|
||
}
|
||
|
||
static int
|
||
get_thaw_percentage()
|
||
{
|
||
uint64_t processes_frozen, processes_thawed;
|
||
processes_frozen = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_frozen, relaxed);
|
||
processes_thawed = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed, relaxed);
|
||
return calculate_thaw_percentage(processes_frozen, processes_thawed);
|
||
}
|
||
|
||
static int
|
||
sysctl_memorystatus_freezer_thaw_percentage SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int thaw_percentage = get_thaw_percentage();
|
||
return sysctl_handle_int(oidp, &thaw_percentage, 0, req);
|
||
}
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freezer_thaw_percentage, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, &sysctl_memorystatus_freezer_thaw_percentage, "I", "");
|
||
|
||
static int
|
||
get_thaw_percentage_fg()
|
||
{
|
||
uint64_t processes_frozen, processes_thawed_fg;
|
||
processes_frozen = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_frozen, relaxed);
|
||
processes_thawed_fg = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed_fg, relaxed);
|
||
return calculate_thaw_percentage(processes_frozen, processes_thawed_fg);
|
||
}
|
||
|
||
static int sysctl_memorystatus_freezer_thaw_percentage_fg SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int thaw_percentage = get_thaw_percentage_fg();
|
||
return sysctl_handle_int(oidp, &thaw_percentage, 0, req);
|
||
}
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freezer_thaw_percentage_fg, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, &sysctl_memorystatus_freezer_thaw_percentage_fg, "I", "");
|
||
|
||
static int
|
||
get_thaw_percentage_webcontent()
|
||
{
|
||
uint64_t processes_frozen_webcontent, processes_thawed_webcontent;
|
||
processes_frozen_webcontent = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_frozen_webcontent, relaxed);
|
||
processes_thawed_webcontent = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed_webcontent, relaxed);
|
||
return calculate_thaw_percentage(processes_frozen_webcontent, processes_thawed_webcontent);
|
||
}
|
||
|
||
static int sysctl_memorystatus_freezer_thaw_percentage_webcontent SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int thaw_percentage = get_thaw_percentage_webcontent();
|
||
return sysctl_handle_int(oidp, &thaw_percentage, 0, req);
|
||
}
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freezer_thaw_percentage_webcontent, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, &sysctl_memorystatus_freezer_thaw_percentage_webcontent, "I", "");
|
||
|
||
|
||
static int
|
||
get_thaw_percentage_bg()
|
||
{
|
||
uint64_t processes_frozen, processes_thawed_fg, processes_thawed;
|
||
processes_frozen = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_frozen, relaxed);
|
||
processes_thawed = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed, relaxed);
|
||
processes_thawed_fg = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed_fg, relaxed);
|
||
return calculate_thaw_percentage(processes_frozen, processes_thawed - processes_thawed_fg);
|
||
}
|
||
|
||
static int sysctl_memorystatus_freezer_thaw_percentage_bg SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int thaw_percentage = get_thaw_percentage_bg();
|
||
return sysctl_handle_int(oidp, &thaw_percentage, 0, req);
|
||
}
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freezer_thaw_percentage_bg, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, &sysctl_memorystatus_freezer_thaw_percentage_bg, "I", "");
|
||
|
||
static int
|
||
get_thaw_percentage_fg_non_xpc_service()
|
||
{
|
||
uint64_t processes_frozen, processes_frozen_xpc_service, processes_thawed_fg, processes_thawed_fg_xpc_service;
|
||
processes_frozen = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_frozen, relaxed);
|
||
processes_frozen_xpc_service = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_frozen_xpc_service, relaxed);
|
||
processes_thawed_fg = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed_fg, relaxed);
|
||
processes_thawed_fg_xpc_service = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed_fg_xpc_service, relaxed);
|
||
/*
|
||
* Since these are all relaxed loads, it's possible (although unlikely) to read a value for
|
||
* frozen/thawed xpc services that's > the value for processes frozen / thawed.
|
||
* Clamp just in case.
|
||
*/
|
||
processes_frozen_xpc_service = MIN(processes_frozen_xpc_service, processes_frozen);
|
||
processes_thawed_fg_xpc_service = MIN(processes_thawed_fg_xpc_service, processes_thawed_fg);
|
||
return calculate_thaw_percentage(processes_frozen - processes_frozen_xpc_service, processes_thawed_fg - processes_thawed_fg_xpc_service);
|
||
}
|
||
|
||
static int sysctl_memorystatus_freezer_thaw_percentage_fg_non_xpc_service SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int thaw_percentage = get_thaw_percentage_fg_non_xpc_service();
|
||
return sysctl_handle_int(oidp, &thaw_percentage, 0, req);
|
||
}
|
||
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freezer_thaw_percentage_fg_non_xpc_service, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, &sysctl_memorystatus_freezer_thaw_percentage_fg_non_xpc_service, "I", "");
|
||
|
||
#define FREEZER_ERROR_STRING_LENGTH 128
|
||
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freeze_pages_min, &memorystatus_freeze_pages_min, 0, UINT32_MAX, "");
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freeze_pages_max, &memorystatus_freeze_pages_max, 0, UINT32_MAX, "");
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freeze_processes_max, &memorystatus_frozen_processes_max, 0, UINT32_MAX, "");
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freeze_jetsam_band, &memorystatus_freeze_jetsam_band, JETSAM_PRIORITY_BACKGROUND, JETSAM_PRIORITY_FOREGROUND, "");
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freeze_private_shared_pages_ratio, &memorystatus_freeze_private_shared_pages_ratio, 0, UINT32_MAX, "");
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freeze_min_processes, &memorystatus_freeze_suspended_threshold, 0, UINT32_MAX, "");
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_freeze_max_candidate_band, &memorystatus_freeze_max_candidate_band, JETSAM_PRIORITY_IDLE, JETSAM_PRIORITY_FOREGROUND, "");
|
||
static int
|
||
sysctl_memorystatus_freeze_budget_multiplier SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2, oidp, req)
|
||
int error = 0, changed = 0;
|
||
uint64_t val = memorystatus_freeze_budget_multiplier;
|
||
unsigned int new_budget;
|
||
clock_sec_t sec;
|
||
clock_nsec_t nsec;
|
||
mach_timespec_t now_ts;
|
||
|
||
error = sysctl_io_number(req, memorystatus_freeze_budget_multiplier, sizeof(val), &val, &changed);
|
||
if (error) {
|
||
return error;
|
||
}
|
||
if (changed) {
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return ENOTSUP;
|
||
}
|
||
#if !(DEVELOPMENT || DEBUG)
|
||
if (val > 100) {
|
||
/* Can not increase budget on release. */
|
||
return EINVAL;
|
||
}
|
||
#endif
|
||
lck_mtx_lock(&freezer_mutex);
|
||
|
||
memorystatus_freeze_budget_multiplier = val;
|
||
/* Start a new throttle interval with this budget multiplier */
|
||
new_budget = memorystatus_freeze_calculate_new_budget(0, 1, NORMAL_WINDOW_MINS, 0);
|
||
clock_get_system_nanotime(&sec, &nsec);
|
||
now_ts.tv_sec = (unsigned int)(MIN(sec, UINT32_MAX));
|
||
now_ts.tv_nsec = nsec;
|
||
memorystatus_freeze_start_normal_throttle_interval(new_budget, now_ts);
|
||
memorystatus_freeze_budget_pages_remaining = normal_throttle_window->max_pageouts;
|
||
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
}
|
||
return 0;
|
||
}
|
||
EXPERIMENT_FACTOR_PROC(_kern, memorystatus_freeze_budget_multiplier, CTLTYPE_QUAD | CTLFLAG_RW, 0, 0, &sysctl_memorystatus_freeze_budget_multiplier, "Q", "");
|
||
/*
|
||
* max. # of frozen process demotions we will allow in our daily cycle.
|
||
*/
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_max_freeze_demotions_daily, &memorystatus_max_frozen_demotions_daily, 0, UINT32_MAX, "");
|
||
|
||
/*
|
||
* min # of thaws needed by a process to protect it from getting demoted into the IDLE band.
|
||
*/
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_thaw_count_demotion_threshold, &memorystatus_thaw_count_demotion_threshold, 0, UINT32_MAX, "");
|
||
|
||
/*
|
||
* min # of global thaws needed for us to consider refreezing these processes.
|
||
*/
|
||
EXPERIMENT_FACTOR_UINT(_kern, memorystatus_min_thaw_refreeze_threshold, &memorystatus_min_thaw_refreeze_threshold, 0, UINT32_MAX, "");
|
||
|
||
#if DEVELOPMENT || DEBUG
|
||
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_daily_mb_max, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_freeze_daily_mb_max, 0, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_degraded_mode, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_freeze_degradation, 0, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_threshold, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_freeze_threshold, 0, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_refreeze_eligible_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_refreeze_eligible_count, 0, "");
|
||
|
||
/*
|
||
* Max. shared-anonymous memory in MB that can be held by frozen processes in the high jetsam band.
|
||
* "0" means no limit.
|
||
* Default is 10% of system-wide task limit.
|
||
*/
|
||
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_shared_mb_max, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_frozen_shared_mb_max, 0, "");
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_shared_mb, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_frozen_shared_mb, 0, "");
|
||
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_shared_mb_per_process_max, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_freeze_shared_mb_per_process_max, 0, "");
|
||
|
||
boolean_t memorystatus_freeze_throttle_enabled = TRUE;
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_throttle_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_freeze_throttle_enabled, 0, "");
|
||
|
||
/*
|
||
* When set to true, this keeps frozen processes in the compressor pool in memory, instead of swapping them out to disk.
|
||
* Exposed via the sysctl kern.memorystatus_freeze_to_memory.
|
||
*/
|
||
boolean_t memorystatus_freeze_to_memory = FALSE;
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_to_memory, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_freeze_to_memory, 0, "");
|
||
|
||
#define VM_PAGES_FOR_ALL_PROCS (2)
|
||
|
||
/*
|
||
* Manual trigger of freeze and thaw for dev / debug kernels only.
|
||
*/
|
||
static int
|
||
sysctl_memorystatus_freeze SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int error, pid = 0;
|
||
proc_t p;
|
||
freezer_error_code_t freezer_error_code = 0;
|
||
pid_t pid_list[MAX_XPC_SERVICE_PIDS];
|
||
int ntasks = 0;
|
||
coalition_t coal = COALITION_NULL;
|
||
|
||
error = sysctl_handle_int(oidp, &pid, 0, req);
|
||
if (error || !req->newptr) {
|
||
return error;
|
||
}
|
||
|
||
if (pid == VM_PAGES_FOR_ALL_PROCS) {
|
||
vm_pageout_anonymous_pages();
|
||
|
||
return 0;
|
||
}
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
if (memorystatus_freeze_enabled == false) {
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
memorystatus_log("sysctl_freeze: Freeze is DISABLED\n");
|
||
return ENOTSUP;
|
||
}
|
||
|
||
again:
|
||
p = proc_find(pid);
|
||
if (p != NULL) {
|
||
memorystatus_freezer_stats.mfs_process_considered_count++;
|
||
uint32_t purgeable, wired, clean, dirty, shared;
|
||
uint32_t max_pages = 0, state = 0;
|
||
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
/*
|
||
* Freezer backed by the compressor and swap file(s)
|
||
* will hold compressed data.
|
||
*
|
||
* Set the sysctl kern.memorystatus_freeze_to_memory to true to keep compressed data from
|
||
* being swapped out to disk. Note that this disables freezer swap support globally,
|
||
* not just for the process being frozen.
|
||
*
|
||
*
|
||
* We don't care about the global freezer budget or the process's (min/max) budget here.
|
||
* The freeze sysctl is meant to force-freeze a process.
|
||
*
|
||
* We also don't update any global or process stats on this path, so that the jetsam/ freeze
|
||
* logic remains unaffected. The tasks we're performing here are: freeze the process, set the
|
||
* P_MEMSTAT_FROZEN bit, and elevate the process to a higher band (if the freezer is active).
|
||
*/
|
||
max_pages = memorystatus_freeze_pages_max;
|
||
} else {
|
||
/*
|
||
* We only have the compressor without any swap.
|
||
*/
|
||
max_pages = UINT32_MAX - 1;
|
||
}
|
||
|
||
proc_list_lock();
|
||
state = p->p_memstat_state;
|
||
proc_list_unlock();
|
||
|
||
/*
|
||
* The jetsam path also verifies that the process is a suspended App. We don't care about that here.
|
||
* We simply ensure that jetsam is not already working on the process and that the process has not
|
||
* explicitly disabled freezing.
|
||
*/
|
||
if (state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_FREEZE_DISABLED)) {
|
||
memorystatus_log_error("sysctl_freeze: p_memstat_state check failed, process is%s%s%s\n",
|
||
(state & P_MEMSTAT_TERMINATED) ? " terminated" : "",
|
||
(state & P_MEMSTAT_LOCKED) ? " locked" : "",
|
||
(state & P_MEMSTAT_FREEZE_DISABLED) ? " unfreezable" : "");
|
||
|
||
proc_rele(p);
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
return EPERM;
|
||
}
|
||
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE) | DBG_FUNC_START, memorystatus_available_pages, pid, max_pages);
|
||
error = task_freeze(proc_task(p), &purgeable, &wired, &clean, &dirty, max_pages, &shared, &freezer_error_code, FALSE /* eval only */);
|
||
if (!error || freezer_error_code == FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO) {
|
||
memorystatus_freezer_stats.mfs_shared_pages_skipped += shared;
|
||
}
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE) | DBG_FUNC_END, purgeable, wired, clean, dirty);
|
||
|
||
if (error) {
|
||
memorystatus_freeze_handle_error(p, freezer_error_code, state & P_MEMSTAT_FROZEN, pid, coal, "sysctl_freeze");
|
||
if (error == KERN_NO_SPACE) {
|
||
/* Make it easy to distinguish between failures due to low compressor/ swap space and other failures. */
|
||
error = ENOSPC;
|
||
} else {
|
||
error = EIO;
|
||
}
|
||
} else {
|
||
proc_list_lock();
|
||
if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == 0) {
|
||
p->p_memstat_state |= P_MEMSTAT_FROZEN;
|
||
p->p_memstat_freeze_skip_reason = kMemorystatusFreezeSkipReasonNone;
|
||
memorystatus_frozen_count++;
|
||
os_atomic_inc(&memorystatus_freezer_stats.mfs_processes_frozen, relaxed);
|
||
if (strcmp(p->p_name, "com.apple.WebKit.WebContent") == 0) {
|
||
memorystatus_frozen_count_webcontent++;
|
||
os_atomic_inc(&(memorystatus_freezer_stats.mfs_processes_frozen_webcontent), relaxed);
|
||
}
|
||
if (memorystatus_frozen_count == memorystatus_frozen_processes_max) {
|
||
memorystatus_freeze_out_of_slots();
|
||
}
|
||
} else {
|
||
// This was a re-freeze
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
memorystatus_freezer_stats.mfs_bytes_refrozen += dirty * PAGE_SIZE;
|
||
memorystatus_freezer_stats.mfs_refreeze_count++;
|
||
}
|
||
}
|
||
p->p_memstat_frozen_count++;
|
||
|
||
if (coal != NULL) {
|
||
/* We just froze an xpc service. Mark it as such for telemetry */
|
||
p->p_memstat_state |= P_MEMSTAT_FROZEN_XPC_SERVICE;
|
||
memorystatus_frozen_count_xpc_service++;
|
||
os_atomic_inc(&(memorystatus_freezer_stats.mfs_processes_frozen_xpc_service), relaxed);
|
||
}
|
||
|
||
|
||
proc_list_unlock();
|
||
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
/*
|
||
* We elevate only if we are going to swap out the data.
|
||
*/
|
||
error = memorystatus_update_inactive_jetsam_priority_band(pid, MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE,
|
||
memorystatus_freeze_jetsam_band, TRUE);
|
||
|
||
if (error) {
|
||
memorystatus_log_error("sysctl_freeze: Elevating frozen process to higher jetsam band failed with %d\n", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
if ((error == 0) && (coal == NULL)) {
|
||
/*
|
||
* We froze a process and so we check to see if it was
|
||
* a coalition leader and if it has XPC services that
|
||
* might need freezing.
|
||
* Only one leader can be frozen at a time and so we shouldn't
|
||
* enter this block more than once per call. Hence the
|
||
* check that 'coal' has to be NULL. We should make this an
|
||
* assert() or panic() once we have a much more concrete way
|
||
* to detect an app vs a daemon.
|
||
*/
|
||
|
||
task_t curr_task = NULL;
|
||
|
||
curr_task = proc_task(p);
|
||
coal = task_get_coalition(curr_task, COALITION_TYPE_JETSAM);
|
||
if (coalition_is_leader(curr_task, coal)) {
|
||
ntasks = coalition_get_pid_list(coal, COALITION_ROLEMASK_XPC,
|
||
COALITION_SORT_DEFAULT, pid_list, MAX_XPC_SERVICE_PIDS);
|
||
|
||
if (ntasks > MAX_XPC_SERVICE_PIDS) {
|
||
ntasks = MAX_XPC_SERVICE_PIDS;
|
||
}
|
||
}
|
||
}
|
||
|
||
proc_rele(p);
|
||
|
||
while (ntasks) {
|
||
pid = pid_list[--ntasks];
|
||
goto again;
|
||
}
|
||
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
return error;
|
||
} else {
|
||
memorystatus_log_error("sysctl_freeze: Invalid process\n");
|
||
}
|
||
|
||
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
return EINVAL;
|
||
}
|
||
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_freeze, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
|
||
0, 0, &sysctl_memorystatus_freeze, "I", "");
|
||
|
||
/*
|
||
* Manual trigger of agressive frozen demotion for dev / debug kernels only.
|
||
*/
|
||
static int
|
||
sysctl_memorystatus_demote_frozen_process SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int error, val;
|
||
/*
|
||
* Only demote on write to prevent demoting during `sysctl -a`.
|
||
* The actual value written doesn't matter.
|
||
*/
|
||
error = sysctl_handle_int(oidp, &val, 0, req);
|
||
if (error || !req->newptr) {
|
||
return error;
|
||
}
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return ENOTSUP;
|
||
}
|
||
lck_mtx_lock(&freezer_mutex);
|
||
memorystatus_demote_frozen_processes(false);
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
return 0;
|
||
}
|
||
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_demote_frozen_processes, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED, 0, 0, &sysctl_memorystatus_demote_frozen_process, "I", "");
|
||
|
||
static int
|
||
sysctl_memorystatus_available_pages_thaw SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
|
||
int error, pid = 0;
|
||
proc_t p;
|
||
|
||
if (memorystatus_freeze_enabled == false) {
|
||
return ENOTSUP;
|
||
}
|
||
|
||
error = sysctl_handle_int(oidp, &pid, 0, req);
|
||
if (error || !req->newptr) {
|
||
return error;
|
||
}
|
||
|
||
if (pid == VM_PAGES_FOR_ALL_PROCS) {
|
||
do_fastwake_warmup_all();
|
||
return 0;
|
||
} else {
|
||
p = proc_find(pid);
|
||
if (p != NULL) {
|
||
error = task_thaw(proc_task(p));
|
||
|
||
if (error) {
|
||
error = EIO;
|
||
} else {
|
||
/*
|
||
* task_thaw() succeeded.
|
||
*
|
||
* We increment memorystatus_frozen_count on the sysctl freeze path.
|
||
* And so we need the P_MEMSTAT_FROZEN to decrement the frozen count
|
||
* when this process exits.
|
||
*
|
||
* proc_list_lock();
|
||
* p->p_memstat_state &= ~P_MEMSTAT_FROZEN;
|
||
* proc_list_unlock();
|
||
*/
|
||
}
|
||
proc_rele(p);
|
||
return error;
|
||
}
|
||
}
|
||
|
||
return EINVAL;
|
||
}
|
||
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_thaw, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
|
||
0, 0, &sysctl_memorystatus_available_pages_thaw, "I", "");
|
||
|
||
|
||
typedef struct _global_freezable_status {
|
||
boolean_t freeze_pages_threshold_crossed;
|
||
boolean_t freeze_eligible_procs_available;
|
||
boolean_t freeze_scheduled_in_future;
|
||
}global_freezable_status_t;
|
||
|
||
typedef struct _proc_freezable_status {
|
||
boolean_t freeze_has_memstat_state;
|
||
boolean_t freeze_has_pages_min;
|
||
int freeze_has_probability;
|
||
int freeze_leader_eligible;
|
||
boolean_t freeze_attempted;
|
||
uint32_t p_memstat_state;
|
||
uint32_t p_pages;
|
||
int p_freeze_error_code;
|
||
int p_pid;
|
||
int p_leader_pid;
|
||
char p_name[MAXCOMLEN + 1];
|
||
}proc_freezable_status_t;
|
||
|
||
#define MAX_FREEZABLE_PROCESSES 200 /* Total # of processes in band 0 that we evaluate for freezability */
|
||
|
||
/*
|
||
* For coalition based freezing evaluations, we proceed as follows:
|
||
* - detect that the process is a coalition member and a XPC service
|
||
* - mark its 'freeze_leader_eligible' field with FREEZE_PROC_LEADER_FREEZABLE_UNKNOWN
|
||
* - continue its freezability evaluation assuming its leader will be freezable too
|
||
*
|
||
* Once we are done evaluating all processes, we do a quick run thru all
|
||
* processes and for a coalition member XPC service we look up the 'freezable'
|
||
* status of its leader and iff:
|
||
* - the xpc service is freezable i.e. its individual freeze evaluation worked
|
||
* - and, its leader is also marked freezable
|
||
* we update its 'freeze_leader_eligible' to FREEZE_PROC_LEADER_FREEZABLE_SUCCESS.
|
||
*/
|
||
|
||
#define FREEZE_PROC_LEADER_FREEZABLE_UNKNOWN (-1)
|
||
#define FREEZE_PROC_LEADER_FREEZABLE_SUCCESS (1)
|
||
#define FREEZE_PROC_LEADER_FREEZABLE_FAILURE (2)
|
||
|
||
static int
|
||
memorystatus_freezer_get_status(user_addr_t buffer, size_t buffer_size, int32_t *retval)
|
||
{
|
||
uint32_t proc_count = 0, freeze_eligible_proc_considered = 0, band = 0, xpc_index = 0, leader_index = 0;
|
||
global_freezable_status_t *list_head;
|
||
proc_freezable_status_t *list_entry, *list_entry_start;
|
||
size_t list_size = 0, entry_count = 0;
|
||
proc_t p, leader_proc;
|
||
memstat_bucket_t *bucket;
|
||
uint32_t state = 0, pages = 0;
|
||
boolean_t try_freeze = TRUE, xpc_skip_size_probability_check = FALSE;
|
||
int error = 0, probability_of_use = 0;
|
||
pid_t leader_pid = 0;
|
||
struct memorystatus_freeze_list_iterator iterator;
|
||
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE == FALSE) {
|
||
return ENOTSUP;
|
||
}
|
||
|
||
bzero(&iterator, sizeof(struct memorystatus_freeze_list_iterator));
|
||
|
||
list_size = sizeof(global_freezable_status_t) + (sizeof(proc_freezable_status_t) * MAX_FREEZABLE_PROCESSES);
|
||
|
||
if (buffer_size < list_size) {
|
||
return EINVAL;
|
||
}
|
||
|
||
list_head = (global_freezable_status_t *)kalloc_data(list_size, Z_WAITOK | Z_ZERO);
|
||
if (list_head == NULL) {
|
||
return ENOMEM;
|
||
}
|
||
|
||
list_size = sizeof(global_freezable_status_t);
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
proc_list_lock();
|
||
|
||
uint64_t curr_time = mach_absolute_time();
|
||
|
||
list_head->freeze_pages_threshold_crossed = (memorystatus_available_pages < memorystatus_freeze_threshold);
|
||
if (memorystatus_freezer_use_ordered_list) {
|
||
list_head->freeze_eligible_procs_available = memorystatus_frozen_count < memorystatus_global_freeze_list.mfcl_length;
|
||
} else {
|
||
list_head->freeze_eligible_procs_available = ((memorystatus_suspended_count - memorystatus_frozen_count) > memorystatus_freeze_suspended_threshold);
|
||
}
|
||
list_head->freeze_scheduled_in_future = (curr_time < memorystatus_freezer_thread_next_run_ts);
|
||
|
||
list_entry_start = (proc_freezable_status_t*) ((uintptr_t)list_head + sizeof(global_freezable_status_t));
|
||
list_entry = list_entry_start;
|
||
|
||
bucket = &memstat_bucket[JETSAM_PRIORITY_IDLE];
|
||
|
||
entry_count = (memorystatus_global_probabilities_size / sizeof(memorystatus_internal_probabilities_t));
|
||
|
||
if (memorystatus_freezer_use_ordered_list) {
|
||
while (iterator.global_freeze_list_index < memorystatus_global_freeze_list.mfcl_length) {
|
||
p = memorystatus_freezer_candidate_list_get_proc(
|
||
&memorystatus_global_freeze_list,
|
||
(iterator.global_freeze_list_index)++,
|
||
NULL);
|
||
if (p != PROC_NULL) {
|
||
break;
|
||
}
|
||
}
|
||
} else {
|
||
p = memorystatus_get_first_proc_locked(&band, FALSE);
|
||
}
|
||
|
||
proc_count++;
|
||
|
||
while ((proc_count <= MAX_FREEZABLE_PROCESSES) &&
|
||
(p) &&
|
||
(list_size < buffer_size)) {
|
||
if (isSysProc(p)) {
|
||
/*
|
||
* Daemon:- We will consider freezing it iff:
|
||
* - it belongs to a coalition and the leader is freeze-eligible (delayed evaluation)
|
||
* - its role in the coalition is XPC service.
|
||
*
|
||
* We skip memory size requirements in this case.
|
||
*/
|
||
|
||
coalition_t coal = COALITION_NULL;
|
||
task_t leader_task = NULL, curr_task = NULL;
|
||
int task_role_in_coalition = 0;
|
||
|
||
curr_task = proc_task(p);
|
||
coal = task_get_coalition(curr_task, COALITION_TYPE_JETSAM);
|
||
|
||
if (coal == COALITION_NULL || coalition_is_leader(curr_task, coal)) {
|
||
/*
|
||
* By default, XPC services without an app
|
||
* will be the leader of their own single-member
|
||
* coalition.
|
||
*/
|
||
goto skip_ineligible_xpc;
|
||
}
|
||
|
||
leader_task = coalition_get_leader(coal);
|
||
if (leader_task == TASK_NULL) {
|
||
/*
|
||
* This jetsam coalition is currently leader-less.
|
||
* This could happen if the app died, but XPC services
|
||
* have not yet exited.
|
||
*/
|
||
goto skip_ineligible_xpc;
|
||
}
|
||
|
||
leader_proc = (proc_t)get_bsdtask_info(leader_task);
|
||
task_deallocate(leader_task);
|
||
|
||
if (leader_proc == PROC_NULL) {
|
||
/* leader task is exiting */
|
||
goto skip_ineligible_xpc;
|
||
}
|
||
|
||
task_role_in_coalition = task_coalition_role_for_type(curr_task, COALITION_TYPE_JETSAM);
|
||
|
||
if (task_role_in_coalition == COALITION_TASKROLE_XPC) {
|
||
xpc_skip_size_probability_check = TRUE;
|
||
leader_pid = proc_getpid(leader_proc);
|
||
goto continue_eval;
|
||
}
|
||
|
||
skip_ineligible_xpc:
|
||
p = memorystatus_get_next_proc_locked(&band, p, FALSE);
|
||
proc_count++;
|
||
continue;
|
||
}
|
||
|
||
continue_eval:
|
||
strlcpy(list_entry->p_name, p->p_name, MAXCOMLEN + 1);
|
||
|
||
list_entry->p_pid = proc_getpid(p);
|
||
|
||
state = p->p_memstat_state;
|
||
|
||
if ((state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_FREEZE_DISABLED | P_MEMSTAT_FREEZE_IGNORE)) ||
|
||
!(state & P_MEMSTAT_SUSPENDED)) {
|
||
try_freeze = list_entry->freeze_has_memstat_state = FALSE;
|
||
} else {
|
||
try_freeze = list_entry->freeze_has_memstat_state = TRUE;
|
||
}
|
||
|
||
list_entry->p_memstat_state = state;
|
||
|
||
if (xpc_skip_size_probability_check == TRUE) {
|
||
/*
|
||
* Assuming the coalition leader is freezable
|
||
* we don't care re. minimum pages and probability
|
||
* as long as the process isn't marked P_MEMSTAT_FREEZE_DISABLED.
|
||
* XPC services have to be explicity opted-out of the disabled
|
||
* state. And we checked that state above.
|
||
*/
|
||
list_entry->freeze_has_pages_min = TRUE;
|
||
list_entry->p_pages = -1;
|
||
list_entry->freeze_has_probability = -1;
|
||
|
||
list_entry->freeze_leader_eligible = FREEZE_PROC_LEADER_FREEZABLE_UNKNOWN;
|
||
list_entry->p_leader_pid = leader_pid;
|
||
|
||
xpc_skip_size_probability_check = FALSE;
|
||
} else {
|
||
list_entry->freeze_leader_eligible = FREEZE_PROC_LEADER_FREEZABLE_SUCCESS; /* Apps are freeze eligible and their own leaders. */
|
||
list_entry->p_leader_pid = 0; /* Setting this to 0 signifies this isn't a coalition driven freeze. */
|
||
|
||
memorystatus_get_task_page_counts(proc_task(p), &pages, NULL, NULL);
|
||
if (pages < memorystatus_freeze_pages_min) {
|
||
try_freeze = list_entry->freeze_has_pages_min = FALSE;
|
||
} else {
|
||
list_entry->freeze_has_pages_min = TRUE;
|
||
}
|
||
|
||
list_entry->p_pages = pages;
|
||
|
||
if (entry_count) {
|
||
uint32_t j = 0;
|
||
for (j = 0; j < entry_count; j++) {
|
||
if (strncmp(memorystatus_global_probabilities_table[j].proc_name,
|
||
p->p_name,
|
||
MAXCOMLEN) == 0) {
|
||
probability_of_use = memorystatus_global_probabilities_table[j].use_probability;
|
||
break;
|
||
}
|
||
}
|
||
|
||
list_entry->freeze_has_probability = probability_of_use;
|
||
|
||
try_freeze = ((probability_of_use > 0) && try_freeze);
|
||
} else {
|
||
list_entry->freeze_has_probability = -1;
|
||
}
|
||
}
|
||
|
||
if (try_freeze) {
|
||
uint32_t purgeable, wired, clean, dirty, shared;
|
||
uint32_t max_pages = 0;
|
||
int freezer_error_code = 0;
|
||
|
||
error = task_freeze(proc_task(p), &purgeable, &wired, &clean, &dirty, max_pages, &shared, &freezer_error_code, TRUE /* eval only */);
|
||
|
||
if (error) {
|
||
list_entry->p_freeze_error_code = freezer_error_code;
|
||
}
|
||
|
||
list_entry->freeze_attempted = TRUE;
|
||
}
|
||
|
||
list_entry++;
|
||
freeze_eligible_proc_considered++;
|
||
|
||
list_size += sizeof(proc_freezable_status_t);
|
||
|
||
if (memorystatus_freezer_use_ordered_list) {
|
||
p = PROC_NULL;
|
||
while (iterator.global_freeze_list_index < memorystatus_global_freeze_list.mfcl_length) {
|
||
p = memorystatus_freezer_candidate_list_get_proc(
|
||
&memorystatus_global_freeze_list,
|
||
(iterator.global_freeze_list_index)++,
|
||
NULL);
|
||
if (p != PROC_NULL) {
|
||
break;
|
||
}
|
||
}
|
||
} else {
|
||
p = memorystatus_get_next_proc_locked(&band, p, FALSE);
|
||
}
|
||
|
||
proc_count++;
|
||
}
|
||
|
||
proc_list_unlock();
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
|
||
list_entry = list_entry_start;
|
||
|
||
for (xpc_index = 0; xpc_index < freeze_eligible_proc_considered; xpc_index++) {
|
||
if (list_entry[xpc_index].freeze_leader_eligible == FREEZE_PROC_LEADER_FREEZABLE_UNKNOWN) {
|
||
leader_pid = list_entry[xpc_index].p_leader_pid;
|
||
|
||
leader_proc = proc_find(leader_pid);
|
||
|
||
if (leader_proc) {
|
||
if (leader_proc->p_memstat_state & P_MEMSTAT_FROZEN) {
|
||
/*
|
||
* Leader has already been frozen.
|
||
*/
|
||
list_entry[xpc_index].freeze_leader_eligible = FREEZE_PROC_LEADER_FREEZABLE_SUCCESS;
|
||
proc_rele(leader_proc);
|
||
continue;
|
||
}
|
||
proc_rele(leader_proc);
|
||
}
|
||
|
||
for (leader_index = 0; leader_index < freeze_eligible_proc_considered; leader_index++) {
|
||
if (list_entry[leader_index].p_pid == leader_pid) {
|
||
if (list_entry[leader_index].freeze_attempted && list_entry[leader_index].p_freeze_error_code == 0) {
|
||
list_entry[xpc_index].freeze_leader_eligible = FREEZE_PROC_LEADER_FREEZABLE_SUCCESS;
|
||
} else {
|
||
list_entry[xpc_index].freeze_leader_eligible = FREEZE_PROC_LEADER_FREEZABLE_FAILURE;
|
||
list_entry[xpc_index].p_freeze_error_code = FREEZER_ERROR_GENERIC;
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
|
||
/*
|
||
* Didn't find the leader entry. This might be likely because
|
||
* the leader never made it down to band 0.
|
||
*/
|
||
if (leader_index == freeze_eligible_proc_considered) {
|
||
list_entry[xpc_index].freeze_leader_eligible = FREEZE_PROC_LEADER_FREEZABLE_FAILURE;
|
||
list_entry[xpc_index].p_freeze_error_code = FREEZER_ERROR_GENERIC;
|
||
}
|
||
}
|
||
}
|
||
|
||
buffer_size = MIN(list_size, INT32_MAX);
|
||
|
||
error = copyout(list_head, buffer, buffer_size);
|
||
if (error == 0) {
|
||
*retval = (int32_t) buffer_size;
|
||
} else {
|
||
*retval = 0;
|
||
}
|
||
|
||
list_size = sizeof(global_freezable_status_t) + (sizeof(proc_freezable_status_t) * MAX_FREEZABLE_PROCESSES);
|
||
kfree_data(list_head, list_size);
|
||
|
||
memorystatus_log_debug("memorystatus_freezer_get_status: returning %d (%lu - size)\n", error, (unsigned long)list_size);
|
||
|
||
return error;
|
||
}
|
||
|
||
#endif /* DEVELOPMENT || DEBUG */
|
||
|
||
/*
|
||
* Get a list of all processes in the freezer band which are currently frozen.
|
||
* Used by powerlog to collect analytics on frozen process.
|
||
*/
|
||
static int
|
||
memorystatus_freezer_get_procs(user_addr_t buffer, size_t buffer_size, int32_t *retval)
|
||
{
|
||
global_frozen_procs_t *frozen_procs = NULL;
|
||
uint32_t band = memorystatus_freeze_jetsam_band;
|
||
proc_t p;
|
||
uint32_t state;
|
||
int error;
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE == FALSE) {
|
||
return ENOTSUP;
|
||
}
|
||
if (buffer_size < sizeof(global_frozen_procs_t)) {
|
||
return EINVAL;
|
||
}
|
||
frozen_procs = (global_frozen_procs_t *)kalloc_data(sizeof(global_frozen_procs_t), Z_WAITOK | Z_ZERO);
|
||
if (frozen_procs == NULL) {
|
||
return ENOMEM;
|
||
}
|
||
|
||
proc_list_lock();
|
||
p = memorystatus_get_first_proc_locked(&band, FALSE);
|
||
while (p && frozen_procs->gfp_num_frozen < FREEZER_CONTROL_GET_PROCS_MAX_COUNT) {
|
||
state = p->p_memstat_state;
|
||
if (state & P_MEMSTAT_FROZEN) {
|
||
frozen_procs->gfp_procs[frozen_procs->gfp_num_frozen].fp_pid = proc_getpid(p);
|
||
strlcpy(frozen_procs->gfp_procs[frozen_procs->gfp_num_frozen].fp_name,
|
||
p->p_name, sizeof(proc_name_t));
|
||
frozen_procs->gfp_num_frozen++;
|
||
}
|
||
p = memorystatus_get_next_proc_locked(&band, p, FALSE);
|
||
}
|
||
proc_list_unlock();
|
||
|
||
buffer_size = MIN(buffer_size, sizeof(global_frozen_procs_t));
|
||
error = copyout(frozen_procs, buffer, buffer_size);
|
||
if (error == 0) {
|
||
*retval = (int32_t) buffer_size;
|
||
} else {
|
||
*retval = 0;
|
||
}
|
||
kfree_data(frozen_procs, sizeof(global_frozen_procs_t));
|
||
|
||
return error;
|
||
}
|
||
|
||
/*
|
||
* If dasd is running an experiment that impacts their freezer candidate selection,
|
||
* we record that in our telemetry.
|
||
*/
|
||
static memorystatus_freezer_trial_identifiers_v1 dasd_trial_identifiers;
|
||
|
||
static int
|
||
memorystatus_freezer_set_dasd_trial_identifiers(user_addr_t buffer, size_t buffer_size, int32_t *retval)
|
||
{
|
||
memorystatus_freezer_trial_identifiers_v1 identifiers;
|
||
int error = 0;
|
||
|
||
if (buffer_size != sizeof(identifiers)) {
|
||
return EINVAL;
|
||
}
|
||
error = copyin(buffer, &identifiers, sizeof(identifiers));
|
||
if (error != 0) {
|
||
return error;
|
||
}
|
||
if (identifiers.version != 1) {
|
||
return EINVAL;
|
||
}
|
||
dasd_trial_identifiers = identifiers;
|
||
*retval = 0;
|
||
return error;
|
||
}
|
||
|
||
/*
|
||
* Reset the freezer state by wiping out all suspended frozen apps, clearing
|
||
* per-process freezer state, and starting a fresh interval.
|
||
*/
|
||
static int
|
||
memorystatus_freezer_reset_state(int32_t *retval)
|
||
{
|
||
uint32_t band = JETSAM_PRIORITY_IDLE;
|
||
/* Don't kill above the frozen band */
|
||
uint32_t kMaxBand = memorystatus_freeze_jetsam_band;
|
||
proc_t next_p = PROC_NULL;
|
||
uint64_t new_budget;
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return ENOTSUP;
|
||
}
|
||
|
||
os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_GENERIC);
|
||
if (jetsam_reason == OS_REASON_NULL) {
|
||
memorystatus_log_error("memorystatus_freezer_reset_state -- sync: failed to allocate jetsam reason\n");
|
||
}
|
||
lck_mtx_lock(&freezer_mutex);
|
||
kill_all_frozen_processes(kMaxBand, true, jetsam_reason, NULL);
|
||
proc_list_lock();
|
||
|
||
/*
|
||
* Clear the considered and skip reason flags on all processes
|
||
* so we're starting fresh with the new policy.
|
||
*/
|
||
next_p = memorystatus_get_first_proc_locked(&band, TRUE);
|
||
while (next_p) {
|
||
proc_t p = next_p;
|
||
uint32_t state = p->p_memstat_state;
|
||
next_p = memorystatus_get_next_proc_locked(&band, p, TRUE);
|
||
|
||
if (p->p_memstat_effectivepriority > kMaxBand) {
|
||
break;
|
||
}
|
||
if (state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED)) {
|
||
continue;
|
||
}
|
||
|
||
p->p_memstat_state &= ~(P_MEMSTAT_FREEZE_CONSIDERED);
|
||
p->p_memstat_freeze_skip_reason = kMemorystatusFreezeSkipReasonNone;
|
||
}
|
||
|
||
proc_list_unlock();
|
||
|
||
new_budget = memorystatus_freeze_calculate_new_budget(0, normal_throttle_window->burst_multiple, normal_throttle_window->mins, 0);
|
||
memorystatus_freeze_force_new_interval(new_budget);
|
||
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
*retval = 0;
|
||
return 0;
|
||
}
|
||
|
||
int
|
||
memorystatus_freezer_control(int32_t flags, user_addr_t buffer, size_t buffer_size, int32_t *retval)
|
||
{
|
||
int err = ENOTSUP;
|
||
|
||
#if DEVELOPMENT || DEBUG
|
||
if (flags == FREEZER_CONTROL_GET_STATUS) {
|
||
err = memorystatus_freezer_get_status(buffer, buffer_size, retval);
|
||
}
|
||
#endif /* DEVELOPMENT || DEBUG */
|
||
if (flags == FREEZER_CONTROL_GET_PROCS) {
|
||
err = memorystatus_freezer_get_procs(buffer, buffer_size, retval);
|
||
} else if (flags == FREEZER_CONTROL_SET_DASD_TRIAL_IDENTIFIERS) {
|
||
err = memorystatus_freezer_set_dasd_trial_identifiers(buffer, buffer_size, retval);
|
||
} else if (flags == FREEZER_CONTROL_RESET_STATE) {
|
||
err = memorystatus_freezer_reset_state(retval);
|
||
}
|
||
|
||
return err;
|
||
}
|
||
|
||
extern void vm_swap_consider_defragmenting(int);
|
||
extern void vm_page_reactivate_all_throttled(void);
|
||
|
||
static bool
|
||
kill_all_frozen_processes(uint64_t max_band, bool suspended_only, os_reason_t jetsam_reason, uint64_t *memory_reclaimed_out)
|
||
{
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
|
||
|
||
unsigned int band = 0;
|
||
proc_t p = PROC_NULL, next_p = PROC_NULL;
|
||
pid_t pid = 0;
|
||
bool retval = false, killed = false;
|
||
uint32_t state;
|
||
uint64_t memory_reclaimed = 0, footprint = 0, skips = 0;
|
||
proc_list_lock();
|
||
|
||
band = JETSAM_PRIORITY_IDLE;
|
||
p = PROC_NULL;
|
||
next_p = PROC_NULL;
|
||
|
||
next_p = memorystatus_get_first_proc_locked(&band, TRUE);
|
||
while (next_p) {
|
||
p = next_p;
|
||
next_p = memorystatus_get_next_proc_locked(&band, p, TRUE);
|
||
state = p->p_memstat_state;
|
||
|
||
if (p->p_memstat_effectivepriority > max_band) {
|
||
break;
|
||
}
|
||
|
||
if (!(state & P_MEMSTAT_FROZEN)) {
|
||
continue;
|
||
}
|
||
|
||
if (suspended_only && !(state & P_MEMSTAT_SUSPENDED)) {
|
||
continue;
|
||
}
|
||
|
||
if (state & P_MEMSTAT_ERROR) {
|
||
p->p_memstat_state &= ~P_MEMSTAT_ERROR;
|
||
}
|
||
|
||
if (state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED)) {
|
||
memorystatus_log("memorystatus: Skipping kill of frozen process %s (%d) because it's already exiting.\n", p->p_name, proc_getpid(p));
|
||
skips++;
|
||
continue;
|
||
}
|
||
|
||
footprint = get_task_phys_footprint(proc_task(p));
|
||
pid = proc_getpid(p);
|
||
proc_list_unlock();
|
||
|
||
/* memorystatus_kill_with_jetsam_reason_sync drops a reference. */
|
||
os_reason_ref(jetsam_reason);
|
||
retval = memorystatus_kill_with_jetsam_reason_sync(pid, jetsam_reason);
|
||
if (retval) {
|
||
killed = true;
|
||
memory_reclaimed += footprint;
|
||
}
|
||
proc_list_lock();
|
||
/*
|
||
* The bands might have changed when we dropped the proc list lock.
|
||
* So start from the beginning.
|
||
* Since we're preventing any further freezing by holding the freezer mutex,
|
||
* and we skip anything we've already tried to kill this is guaranteed to terminate.
|
||
*/
|
||
band = 0;
|
||
skips = 0;
|
||
next_p = memorystatus_get_first_proc_locked(&band, TRUE);
|
||
}
|
||
|
||
assert(skips <= memorystatus_frozen_count);
|
||
#if DEVELOPMENT || DEBUG
|
||
if (!suspended_only && max_band >= JETSAM_PRIORITY_FOREGROUND) {
|
||
/*
|
||
* Check that we've killed all frozen processes.
|
||
* Note that they may still be exiting (represented by skips).
|
||
*/
|
||
if (memorystatus_frozen_count - skips > 0) {
|
||
assert(memorystatus_freeze_enabled == false);
|
||
|
||
panic("memorystatus_disable_freeze: Failed to kill all frozen processes, memorystatus_frozen_count = %d",
|
||
memorystatus_frozen_count);
|
||
}
|
||
}
|
||
#endif /* DEVELOPMENT || DEBUG */
|
||
if (memory_reclaimed_out) {
|
||
*memory_reclaimed_out = memory_reclaimed;
|
||
}
|
||
proc_list_unlock();
|
||
return killed;
|
||
}
|
||
|
||
/*
|
||
* Disables the freezer, jetsams all frozen processes,
|
||
* and reclaims the swap space immediately.
|
||
*/
|
||
|
||
void
|
||
memorystatus_disable_freeze(void)
|
||
{
|
||
uint64_t memory_reclaimed = 0;
|
||
bool killed = false;
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
|
||
|
||
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE_DISABLE) | DBG_FUNC_START,
|
||
memorystatus_available_pages);
|
||
memorystatus_log("memorystatus: Disabling freezer. Will kill all frozen processes\n");
|
||
|
||
/*
|
||
* We hold the freezer_mutex (preventing anything from being frozen in parallel)
|
||
* and all frozen processes will be killed
|
||
* by the time we release it. Setting memorystatus_freeze_enabled to false,
|
||
* ensures that no new processes will be frozen once we release the mutex.
|
||
*
|
||
*/
|
||
memorystatus_freeze_enabled = false;
|
||
|
||
/*
|
||
* Move dirty pages out from the throttle to the active queue since we're not freezing anymore.
|
||
*/
|
||
vm_page_reactivate_all_throttled();
|
||
os_reason_t jetsam_reason = os_reason_create(OS_REASON_JETSAM, JETSAM_REASON_MEMORY_DISK_SPACE_SHORTAGE);
|
||
if (jetsam_reason == OS_REASON_NULL) {
|
||
memorystatus_log_error("memorystatus_disable_freeze -- sync: failed to allocate jetsam reason\n");
|
||
}
|
||
|
||
killed = kill_all_frozen_processes(JETSAM_PRIORITY_FOREGROUND, false, jetsam_reason, &memory_reclaimed);
|
||
|
||
if (killed) {
|
||
memorystatus_log_info("memorystatus: Killed all frozen processes.\n");
|
||
vm_swap_consider_defragmenting(VM_SWAP_FLAGS_FORCE_DEFRAG | VM_SWAP_FLAGS_FORCE_RECLAIM);
|
||
|
||
proc_list_lock();
|
||
size_t snapshot_size = sizeof(memorystatus_jetsam_snapshot_t) +
|
||
sizeof(memorystatus_jetsam_snapshot_entry_t) * (memorystatus_jetsam_snapshot_count);
|
||
uint64_t timestamp_now = mach_absolute_time();
|
||
memorystatus_jetsam_snapshot->notification_time = timestamp_now;
|
||
memorystatus_jetsam_snapshot->js_gencount++;
|
||
if (memorystatus_jetsam_snapshot_count > 0 && (memorystatus_jetsam_snapshot_last_timestamp == 0 ||
|
||
timestamp_now > memorystatus_jetsam_snapshot_last_timestamp + memorystatus_jetsam_snapshot_timeout)) {
|
||
proc_list_unlock();
|
||
int ret = memorystatus_send_note(kMemorystatusSnapshotNote, &snapshot_size, sizeof(snapshot_size));
|
||
if (!ret) {
|
||
proc_list_lock();
|
||
memorystatus_jetsam_snapshot_last_timestamp = timestamp_now;
|
||
}
|
||
}
|
||
proc_list_unlock();
|
||
} else {
|
||
memorystatus_log_info("memorystatus: No frozen processes to kill.\n");
|
||
}
|
||
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE_DISABLE) | DBG_FUNC_END,
|
||
memorystatus_available_pages, memory_reclaimed);
|
||
|
||
return;
|
||
}
|
||
|
||
static void
|
||
memorystatus_set_freeze_is_enabled(bool enabled)
|
||
{
|
||
lck_mtx_lock(&freezer_mutex);
|
||
if (enabled != memorystatus_freeze_enabled) {
|
||
if (enabled) {
|
||
memorystatus_freeze_enabled = true;
|
||
} else {
|
||
memorystatus_disable_freeze();
|
||
}
|
||
}
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
}
|
||
|
||
|
||
static int
|
||
sysctl_freeze_enabled SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int error, val = memorystatus_freeze_enabled ? 1 : 0;
|
||
|
||
error = sysctl_handle_int(oidp, &val, 0, req);
|
||
if (error || !req->newptr) {
|
||
return error;
|
||
}
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
memorystatus_log_error("memorystatus: Failed attempt to set vm.freeze_enabled sysctl\n");
|
||
return EINVAL;
|
||
}
|
||
|
||
memorystatus_set_freeze_is_enabled(val);
|
||
|
||
return 0;
|
||
}
|
||
|
||
SYSCTL_PROC(_vm, OID_AUTO, freeze_enabled, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY, NULL, 0, sysctl_freeze_enabled, "I", "");
|
||
|
||
static void
|
||
schedule_interval_reset(thread_call_t reset_thread_call, throttle_interval_t *interval)
|
||
{
|
||
uint64_t interval_expiration_ns = interval->ts.tv_sec * NSEC_PER_SEC + interval->ts.tv_nsec;
|
||
uint64_t interval_expiration_absolutetime;
|
||
nanoseconds_to_absolutetime(interval_expiration_ns, &interval_expiration_absolutetime);
|
||
memorystatus_log_info("memorystatus: scheduling new freezer interval at %llu absolute time\n", interval_expiration_absolutetime);
|
||
|
||
thread_call_enter_delayed(reset_thread_call, interval_expiration_absolutetime);
|
||
}
|
||
|
||
extern uuid_string_t trial_treatment_id;
|
||
extern uuid_string_t trial_experiment_id;
|
||
extern int trial_deployment_id;
|
||
|
||
CA_EVENT(freezer_interval,
|
||
CA_INT, budget_remaining,
|
||
CA_INT, error_below_min_pages,
|
||
CA_INT, error_excess_shared_memory,
|
||
CA_INT, error_low_private_shared_ratio,
|
||
CA_INT, error_no_compressor_space,
|
||
CA_INT, error_no_swap_space,
|
||
CA_INT, error_low_probability_of_use,
|
||
CA_INT, error_elevated,
|
||
CA_INT, error_other,
|
||
CA_INT, frozen_count,
|
||
CA_INT, pageouts,
|
||
CA_INT, refreeze_average,
|
||
CA_INT, skipped_full,
|
||
CA_INT, skipped_shared_mb_high,
|
||
CA_INT, swapusage,
|
||
CA_INT, thaw_count,
|
||
CA_INT, thaw_percentage,
|
||
CA_INT, thaws_per_gb,
|
||
CA_INT, trial_deployment_id,
|
||
CA_INT, dasd_trial_deployment_id,
|
||
CA_INT, budget_exhaustion_duration_remaining,
|
||
CA_INT, thaw_percentage_webcontent,
|
||
CA_INT, thaw_percentage_fg,
|
||
CA_INT, thaw_percentage_bg,
|
||
CA_INT, thaw_percentage_fg_non_xpc_service,
|
||
CA_INT, fg_resume_count,
|
||
CA_INT, unique_freeze_count,
|
||
CA_INT, unique_thaw_count,
|
||
CA_STATIC_STRING(CA_UUID_LEN), trial_treatment_id,
|
||
CA_STATIC_STRING(CA_UUID_LEN), trial_experiment_id,
|
||
CA_STATIC_STRING(CA_UUID_LEN), dasd_trial_treatment_id,
|
||
CA_STATIC_STRING(CA_UUID_LEN), dasd_trial_experiment_id);
|
||
|
||
extern uint64_t vm_swap_get_total_space(void);
|
||
extern uint64_t vm_swap_get_free_space(void);
|
||
|
||
/*
|
||
* Record statistics from the expiring interval
|
||
* via core analytics.
|
||
*/
|
||
static void
|
||
memorystatus_freeze_record_interval_analytics(void)
|
||
{
|
||
ca_event_t event = CA_EVENT_ALLOCATE(freezer_interval);
|
||
CA_EVENT_TYPE(freezer_interval) * e = event->data;
|
||
e->budget_remaining = memorystatus_freeze_budget_pages_remaining * PAGE_SIZE / (1UL << 20);
|
||
uint64_t process_considered_count, refrozen_count, below_threshold_count;
|
||
memory_object_size_t swap_size;
|
||
process_considered_count = memorystatus_freezer_stats.mfs_process_considered_count;
|
||
if (process_considered_count != 0) {
|
||
e->error_below_min_pages = memorystatus_freezer_stats.mfs_error_below_min_pages_count * 100 / process_considered_count;
|
||
e->error_excess_shared_memory = memorystatus_freezer_stats.mfs_error_excess_shared_memory_count * 100 / process_considered_count;
|
||
e->error_low_private_shared_ratio = memorystatus_freezer_stats.mfs_error_low_private_shared_ratio_count * 100 / process_considered_count;
|
||
e->error_no_compressor_space = memorystatus_freezer_stats.mfs_error_no_compressor_space_count * 100 / process_considered_count;
|
||
e->error_no_swap_space = memorystatus_freezer_stats.mfs_error_no_swap_space_count * 100 / process_considered_count;
|
||
e->error_low_probability_of_use = memorystatus_freezer_stats.mfs_error_low_probability_of_use_count * 100 / process_considered_count;
|
||
e->error_elevated = memorystatus_freezer_stats.mfs_error_elevated_count * 100 / process_considered_count;
|
||
e->error_other = memorystatus_freezer_stats.mfs_error_other_count * 100 / process_considered_count;
|
||
}
|
||
e->frozen_count = memorystatus_frozen_count;
|
||
e->pageouts = normal_throttle_window->pageouts * PAGE_SIZE / (1UL << 20);
|
||
refrozen_count = memorystatus_freezer_stats.mfs_refreeze_count;
|
||
if (refrozen_count != 0) {
|
||
e->refreeze_average = (memorystatus_freezer_stats.mfs_bytes_refrozen / (1UL << 20)) / refrozen_count;
|
||
}
|
||
below_threshold_count = memorystatus_freezer_stats.mfs_below_threshold_count;
|
||
if (below_threshold_count != 0) {
|
||
e->skipped_full = memorystatus_freezer_stats.mfs_skipped_full_count * 100 / below_threshold_count;
|
||
e->skipped_shared_mb_high = memorystatus_freezer_stats.mfs_skipped_shared_mb_high_count * 100 / below_threshold_count;
|
||
}
|
||
if (VM_CONFIG_SWAP_IS_PRESENT) {
|
||
swap_size = vm_swap_get_total_space();
|
||
if (swap_size) {
|
||
e->swapusage = vm_swap_get_free_space() * 100 / swap_size;
|
||
}
|
||
}
|
||
e->thaw_count = memorystatus_thaw_count;
|
||
e->thaw_percentage = get_thaw_percentage();
|
||
e->thaw_percentage_webcontent = get_thaw_percentage_webcontent();
|
||
e->thaw_percentage_fg = get_thaw_percentage_fg();
|
||
e->thaw_percentage_bg = get_thaw_percentage_bg();
|
||
e->thaw_percentage_fg_non_xpc_service = get_thaw_percentage_fg_non_xpc_service();
|
||
|
||
if (e->pageouts / (1UL << 10) != 0) {
|
||
e->thaws_per_gb = memorystatus_thaw_count / (e->pageouts / (1UL << 10));
|
||
}
|
||
e->budget_exhaustion_duration_remaining = memorystatus_freezer_stats.mfs_budget_exhaustion_duration_remaining;
|
||
e->fg_resume_count = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed_fg, relaxed);
|
||
e->unique_freeze_count = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_frozen, relaxed);
|
||
e->unique_thaw_count = os_atomic_load(&memorystatus_freezer_stats.mfs_processes_thawed, relaxed);
|
||
|
||
/*
|
||
* Record any xnu or dasd experiment information
|
||
*/
|
||
strlcpy(e->trial_treatment_id, trial_treatment_id, CA_UUID_LEN);
|
||
strlcpy(e->trial_experiment_id, trial_experiment_id, CA_UUID_LEN);
|
||
e->trial_deployment_id = trial_deployment_id;
|
||
strlcpy(e->dasd_trial_treatment_id, dasd_trial_identifiers.treatment_id, CA_UUID_LEN);
|
||
strlcpy(e->dasd_trial_experiment_id, dasd_trial_identifiers.experiment_id, CA_UUID_LEN);
|
||
e->dasd_trial_deployment_id = dasd_trial_identifiers.deployment_id;
|
||
|
||
CA_EVENT_SEND(event);
|
||
}
|
||
|
||
static void
|
||
memorystatus_freeze_reset_interval(void *arg0, void *arg1)
|
||
{
|
||
#pragma unused(arg0, arg1)
|
||
struct throttle_interval_t *interval = NULL;
|
||
clock_sec_t sec;
|
||
clock_nsec_t nsec;
|
||
mach_timespec_t now_ts;
|
||
uint32_t budget_rollover = 0;
|
||
|
||
clock_get_system_nanotime(&sec, &nsec);
|
||
now_ts.tv_sec = (unsigned int)(MIN(sec, UINT32_MAX));
|
||
now_ts.tv_nsec = nsec;
|
||
interval = normal_throttle_window;
|
||
|
||
/* Record analytics from the old interval before resetting. */
|
||
memorystatus_freeze_record_interval_analytics();
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
/* How long has it been since the previous interval expired? */
|
||
mach_timespec_t expiration_period_ts = now_ts;
|
||
SUB_MACH_TIMESPEC(&expiration_period_ts, &interval->ts);
|
||
/* Get unused budget. Clamp to 0. We'll adjust for overused budget in the next interval. */
|
||
budget_rollover = interval->pageouts > interval->max_pageouts ?
|
||
0 : interval->max_pageouts - interval->pageouts;
|
||
|
||
memorystatus_freeze_start_normal_throttle_interval(memorystatus_freeze_calculate_new_budget(
|
||
expiration_period_ts.tv_sec, interval->burst_multiple,
|
||
interval->mins, budget_rollover),
|
||
now_ts);
|
||
memorystatus_freeze_budget_pages_remaining = interval->max_pageouts;
|
||
|
||
if (!memorystatus_freezer_use_demotion_list) {
|
||
memorystatus_demote_frozen_processes(false); /* normal mode...don't force a demotion */
|
||
}
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
}
|
||
|
||
|
||
proc_t
|
||
memorystatus_get_coalition_leader_and_role(proc_t p, int *role_in_coalition)
|
||
{
|
||
coalition_t coal = COALITION_NULL;
|
||
task_t leader_task = NULL, curr_task = NULL;
|
||
proc_t leader_proc = PROC_NULL;
|
||
|
||
curr_task = proc_task(p);
|
||
coal = task_get_coalition(curr_task, COALITION_TYPE_JETSAM);
|
||
|
||
if (coal == NULL || coalition_is_leader(curr_task, coal)) {
|
||
return p;
|
||
}
|
||
|
||
leader_task = coalition_get_leader(coal);
|
||
if (leader_task == TASK_NULL) {
|
||
/*
|
||
* This jetsam coalition is currently leader-less.
|
||
* This could happen if the app died, but XPC services
|
||
* have not yet exited.
|
||
*/
|
||
return PROC_NULL;
|
||
}
|
||
|
||
leader_proc = (proc_t)get_bsdtask_info(leader_task);
|
||
task_deallocate(leader_task);
|
||
|
||
if (leader_proc == PROC_NULL) {
|
||
/* leader task is exiting */
|
||
return PROC_NULL;
|
||
}
|
||
|
||
*role_in_coalition = task_coalition_role_for_type(curr_task, COALITION_TYPE_JETSAM);
|
||
|
||
return leader_proc;
|
||
}
|
||
|
||
bool
|
||
memorystatus_freeze_process_is_recommended(const proc_t p)
|
||
{
|
||
assert(!memorystatus_freezer_use_ordered_list);
|
||
int probability_of_use = 0;
|
||
|
||
size_t entry_count = 0, i = 0;
|
||
entry_count = (memorystatus_global_probabilities_size / sizeof(memorystatus_internal_probabilities_t));
|
||
if (entry_count == 0) {
|
||
/*
|
||
* If dasd hasn't supplied a table yet, we default to every app being eligible
|
||
* for the freezer.
|
||
*/
|
||
return true;
|
||
}
|
||
for (i = 0; i < entry_count; i++) {
|
||
/*
|
||
* NB: memorystatus_internal_probabilities.proc_name is MAXCOMLEN + 1 bytes
|
||
* proc_t.p_name is 2*MAXCOMLEN + 1 bytes. So we only compare the first
|
||
* MAXCOMLEN bytes here since the name in the probabilities table could
|
||
* be truncated from the proc_t's p_name.
|
||
*/
|
||
if (strncmp(memorystatus_global_probabilities_table[i].proc_name,
|
||
p->p_name,
|
||
MAXCOMLEN) == 0) {
|
||
probability_of_use = memorystatus_global_probabilities_table[i].use_probability;
|
||
break;
|
||
}
|
||
}
|
||
return probability_of_use > 0;
|
||
}
|
||
|
||
__private_extern__ void
|
||
memorystatus_freeze_init(void)
|
||
{
|
||
kern_return_t result;
|
||
thread_t thread;
|
||
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
/*
|
||
* This is just the default value if the underlying
|
||
* storage device doesn't have any specific budget.
|
||
* We check with the storage layer in memorystatus_freeze_update_throttle()
|
||
* before we start our freezing the first time.
|
||
*/
|
||
memorystatus_freeze_budget_pages_remaining = (memorystatus_freeze_daily_mb_max * 1024 * 1024) / PAGE_SIZE;
|
||
|
||
result = kernel_thread_start(memorystatus_freeze_thread, NULL, &thread);
|
||
if (result == KERN_SUCCESS) {
|
||
proc_set_thread_policy(thread, TASK_POLICY_INTERNAL, TASK_POLICY_IO, THROTTLE_LEVEL_COMPRESSOR_TIER2);
|
||
proc_set_thread_policy(thread, TASK_POLICY_INTERNAL, TASK_POLICY_PASSIVE_IO, TASK_POLICY_ENABLE);
|
||
thread_set_thread_name(thread, "VM_freezer");
|
||
|
||
thread_deallocate(thread);
|
||
} else {
|
||
panic("Could not create memorystatus_freeze_thread");
|
||
}
|
||
|
||
freeze_interval_reset_thread_call = thread_call_allocate_with_options(memorystatus_freeze_reset_interval, NULL, THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE);
|
||
/* Start a new interval */
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
uint32_t budget;
|
||
budget = memorystatus_freeze_calculate_new_budget(0, normal_throttle_window->burst_multiple, normal_throttle_window->mins, 0);
|
||
memorystatus_freeze_force_new_interval(budget);
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
} else {
|
||
memorystatus_freeze_budget_pages_remaining = 0;
|
||
}
|
||
}
|
||
|
||
void
|
||
memorystatus_freeze_configure_for_swap()
|
||
{
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return;
|
||
}
|
||
|
||
assert(memorystatus_swap_all_apps);
|
||
|
||
/*
|
||
* We expect both a larger working set and larger individual apps
|
||
* in this mode, so tune up the freezer accordingly.
|
||
*/
|
||
memorystatus_frozen_processes_max = FREEZE_PROCESSES_MAX_SWAP_ENABLED;
|
||
memorystatus_max_frozen_demotions_daily = MAX_FROZEN_PROCESS_DEMOTIONS_SWAP_ENABLED;
|
||
memorystatus_freeze_pages_max = FREEZE_PAGES_MAX_SWAP_ENABLED;
|
||
|
||
/*
|
||
* We don't have a budget when running with full app swap.
|
||
* Force a new interval. memorystatus_freeze_calculate_new_budget should give us an
|
||
* unlimited budget.
|
||
*/
|
||
lck_mtx_lock(&freezer_mutex);
|
||
uint32_t budget;
|
||
budget = memorystatus_freeze_calculate_new_budget(0, normal_throttle_window->burst_multiple, normal_throttle_window->mins, 0);
|
||
memorystatus_freeze_force_new_interval(budget);
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
}
|
||
|
||
void
|
||
memorystatus_freeze_disable_swap()
|
||
{
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return;
|
||
}
|
||
|
||
assert(!memorystatus_swap_all_apps);
|
||
|
||
memorystatus_frozen_processes_max = FREEZE_PROCESSES_MAX;
|
||
memorystatus_max_frozen_demotions_daily = MAX_FROZEN_PROCESS_DEMOTIONS;
|
||
memorystatus_freeze_pages_max = FREEZE_PAGES_MAX;
|
||
|
||
/*
|
||
* Calculate a new budget now that we're constrained by our daily write budget again.
|
||
*/
|
||
lck_mtx_lock(&freezer_mutex);
|
||
uint32_t budget;
|
||
budget = memorystatus_freeze_calculate_new_budget(0, normal_throttle_window->burst_multiple, normal_throttle_window->mins, 0);
|
||
memorystatus_freeze_force_new_interval(budget);
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
}
|
||
|
||
/*
|
||
* Called with both the freezer_mutex and proc_list_lock held & both will be held on return.
|
||
*/
|
||
static int
|
||
memorystatus_freeze_process(
|
||
proc_t p,
|
||
coalition_t *coal, /* IN / OUT */
|
||
pid_t *coalition_list, /* OUT */
|
||
unsigned int *coalition_list_length /* OUT */)
|
||
{
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_OWNED);
|
||
|
||
kern_return_t kr;
|
||
uint32_t purgeable, wired, clean, dirty, shared;
|
||
uint64_t max_pages = 0;
|
||
freezer_error_code_t freezer_error_code = 0;
|
||
bool is_refreeze = false;
|
||
task_t curr_task = TASK_NULL;
|
||
|
||
pid_t aPid = proc_getpid(p);
|
||
|
||
is_refreeze = (p->p_memstat_state & P_MEMSTAT_FROZEN) != 0;
|
||
|
||
/* Ensure the process is eligible for (re-)freezing */
|
||
if (is_refreeze && !memorystatus_freeze_proc_is_refreeze_eligible(p)) {
|
||
/* Process is already frozen & hasn't been thawed. Nothing to do here. */
|
||
return EINVAL;
|
||
}
|
||
if (is_refreeze) {
|
||
/*
|
||
* Not currently being looked at for something.
|
||
*/
|
||
if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
|
||
return EBUSY;
|
||
}
|
||
|
||
/*
|
||
* We are going to try and refreeze and so re-evaluate
|
||
* the process. We don't want to double count the shared
|
||
* memory. So deduct the old snapshot here.
|
||
*/
|
||
memorystatus_frozen_shared_mb -= p->p_memstat_freeze_sharedanon_pages;
|
||
p->p_memstat_freeze_sharedanon_pages = 0;
|
||
|
||
p->p_memstat_state &= ~P_MEMSTAT_REFREEZE_ELIGIBLE;
|
||
memorystatus_refreeze_eligible_count--;
|
||
} else {
|
||
if (!memorystatus_is_process_eligible_for_freeze(p)) {
|
||
return EINVAL;
|
||
}
|
||
if (memorystatus_frozen_count >= memorystatus_frozen_processes_max) {
|
||
memorystatus_freeze_handle_error(p, FREEZER_ERROR_NO_SLOTS, is_refreeze, aPid, (coal ? *coal : NULL), "memorystatus_freeze_process");
|
||
return ENOSPC;
|
||
}
|
||
}
|
||
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
/*
|
||
* Freezer backed by the compressor and swap file(s)
|
||
* will hold compressed data.
|
||
*/
|
||
|
||
max_pages = MIN(memorystatus_freeze_pages_max, memorystatus_freeze_budget_pages_remaining);
|
||
} else {
|
||
/*
|
||
* We only have the compressor pool.
|
||
*/
|
||
max_pages = UINT32_MAX - 1;
|
||
}
|
||
|
||
/* Mark as locked temporarily to avoid kill */
|
||
p->p_memstat_state |= P_MEMSTAT_LOCKED;
|
||
|
||
p = proc_ref(p, true);
|
||
if (!p) {
|
||
memorystatus_freezer_stats.mfs_error_other_count++;
|
||
return EBUSY;
|
||
}
|
||
|
||
proc_list_unlock();
|
||
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE) | DBG_FUNC_START, memorystatus_available_pages, aPid, max_pages);
|
||
|
||
max_pages = MIN(max_pages, UINT32_MAX);
|
||
kr = task_freeze(proc_task(p), &purgeable, &wired, &clean, &dirty, (uint32_t) max_pages, &shared, &freezer_error_code, FALSE /* eval only */);
|
||
if (kr == KERN_SUCCESS || freezer_error_code == FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO) {
|
||
memorystatus_freezer_stats.mfs_shared_pages_skipped += shared;
|
||
}
|
||
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE) | DBG_FUNC_END, purgeable, wired, clean, dirty);
|
||
|
||
memorystatus_log_debug("memorystatus_freeze_top_process: task_freeze %s for pid %d [%s] - "
|
||
"memorystatus_pages: %d, purgeable: %d, wired: %d, clean: %d, dirty: %d, max_pages %llu, shared %d",
|
||
(kr == KERN_SUCCESS) ? "SUCCEEDED" : "FAILED", aPid, (*p->p_name ? p->p_name : "(unknown)"),
|
||
memorystatus_available_pages, purgeable, wired, clean, dirty, max_pages, shared);
|
||
|
||
proc_list_lock();
|
||
|
||
/* Success? */
|
||
if (KERN_SUCCESS == kr) {
|
||
memorystatus_freeze_entry_t data = { aPid, TRUE, dirty };
|
||
|
||
p->p_memstat_freeze_sharedanon_pages += shared;
|
||
|
||
memorystatus_frozen_shared_mb += shared;
|
||
|
||
if (!is_refreeze) {
|
||
p->p_memstat_state |= P_MEMSTAT_FROZEN;
|
||
p->p_memstat_freeze_skip_reason = kMemorystatusFreezeSkipReasonNone;
|
||
memorystatus_frozen_count++;
|
||
os_atomic_inc(&memorystatus_freezer_stats.mfs_processes_frozen, relaxed);
|
||
if (strcmp(p->p_name, "com.apple.WebKit.WebContent") == 0) {
|
||
memorystatus_frozen_count_webcontent++;
|
||
os_atomic_inc(&(memorystatus_freezer_stats.mfs_processes_frozen_webcontent), relaxed);
|
||
}
|
||
if (memorystatus_frozen_count == memorystatus_frozen_processes_max) {
|
||
memorystatus_freeze_out_of_slots();
|
||
}
|
||
} else {
|
||
// This was a re-freeze
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
memorystatus_freezer_stats.mfs_bytes_refrozen += dirty * PAGE_SIZE;
|
||
memorystatus_freezer_stats.mfs_refreeze_count++;
|
||
}
|
||
}
|
||
|
||
p->p_memstat_frozen_count++;
|
||
|
||
/*
|
||
* Still keeping the P_MEMSTAT_LOCKED bit till we are actually done elevating this frozen process
|
||
* to its higher jetsam band.
|
||
*/
|
||
proc_list_unlock();
|
||
|
||
memorystatus_send_note(kMemorystatusFreezeNote, &data, sizeof(data));
|
||
|
||
if (VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
int ret;
|
||
unsigned int i;
|
||
ret = memorystatus_update_inactive_jetsam_priority_band(proc_getpid(p), MEMORYSTATUS_CMD_ELEVATED_INACTIVEJETSAMPRIORITY_ENABLE, memorystatus_freeze_jetsam_band, TRUE);
|
||
|
||
if (ret) {
|
||
memorystatus_log_error("Elevating the frozen process failed with %d\n", ret);
|
||
/* not fatal */
|
||
}
|
||
|
||
/* Update stats */
|
||
for (i = 0; i < sizeof(throttle_intervals) / sizeof(struct throttle_interval_t); i++) {
|
||
throttle_intervals[i].pageouts += dirty;
|
||
}
|
||
}
|
||
memorystatus_freeze_update_throttle(&memorystatus_freeze_budget_pages_remaining);
|
||
memorystatus_log("memorystatus: %sfreezing (%s) pid %d [%s] done, memorystatus_freeze_budget_pages_remaining %llu %sfroze %u pages\n",
|
||
is_refreeze ? "re" : "", ((!coal || !*coal) ? "general" : "coalition-driven"), aPid, ((p && *p->p_name) ? p->p_name : "unknown"),
|
||
memorystatus_freeze_budget_pages_remaining, is_refreeze ? "Re" : "", dirty);
|
||
|
||
proc_list_lock();
|
||
|
||
memorystatus_freeze_pageouts += dirty;
|
||
|
||
if (memorystatus_frozen_count == (memorystatus_frozen_processes_max - 1)) {
|
||
/*
|
||
* Add some eviction logic here? At some point should we
|
||
* jetsam a process to get back its swap space so that we
|
||
* can freeze a more eligible process at this moment in time?
|
||
*/
|
||
}
|
||
|
||
/* Check if we just froze a coalition leader. If so, return the list of XPC services to freeze next. */
|
||
if (coal != NULL && *coal == NULL) {
|
||
curr_task = proc_task(p);
|
||
*coal = task_get_coalition(curr_task, COALITION_TYPE_JETSAM);
|
||
if (coalition_is_leader(curr_task, *coal)) {
|
||
*coalition_list_length = coalition_get_pid_list(*coal, COALITION_ROLEMASK_XPC,
|
||
COALITION_SORT_DEFAULT, coalition_list, MAX_XPC_SERVICE_PIDS);
|
||
|
||
if (*coalition_list_length > MAX_XPC_SERVICE_PIDS) {
|
||
*coalition_list_length = MAX_XPC_SERVICE_PIDS;
|
||
}
|
||
}
|
||
} else {
|
||
/* We just froze an xpc service. Mark it as such for telemetry */
|
||
p->p_memstat_state |= P_MEMSTAT_FROZEN_XPC_SERVICE;
|
||
memorystatus_frozen_count_xpc_service++;
|
||
os_atomic_inc(&(memorystatus_freezer_stats.mfs_processes_frozen_xpc_service), relaxed);
|
||
}
|
||
|
||
p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
|
||
wakeup(&p->p_memstat_state);
|
||
proc_rele(p);
|
||
return 0;
|
||
} else {
|
||
if (is_refreeze) {
|
||
if ((freezer_error_code == FREEZER_ERROR_EXCESS_SHARED_MEMORY) ||
|
||
(freezer_error_code == FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO)) {
|
||
/*
|
||
* Keeping this prior-frozen process in this high band when
|
||
* we failed to re-freeze it due to bad shared memory usage
|
||
* could cause excessive pressure on the lower bands.
|
||
* We need to demote it for now. It'll get re-evaluated next
|
||
* time because we don't set the P_MEMSTAT_FREEZE_IGNORE
|
||
* bit.
|
||
*/
|
||
|
||
p->p_memstat_state &= ~P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
|
||
memorystatus_invalidate_idle_demotion_locked(p, TRUE);
|
||
memorystatus_update_priority_locked(p, JETSAM_PRIORITY_IDLE, TRUE, TRUE);
|
||
}
|
||
} else {
|
||
p->p_memstat_state |= P_MEMSTAT_FREEZE_IGNORE;
|
||
}
|
||
memorystatus_freeze_handle_error(p, freezer_error_code, p->p_memstat_state & P_MEMSTAT_FROZEN, aPid, (coal != NULL) ? *coal : NULL, "memorystatus_freeze_process");
|
||
|
||
p->p_memstat_state &= ~P_MEMSTAT_LOCKED;
|
||
wakeup(&p->p_memstat_state);
|
||
proc_rele(p);
|
||
|
||
return EINVAL;
|
||
}
|
||
}
|
||
|
||
/*
|
||
* Synchronously freeze the passed proc. Called with a reference to the proc held.
|
||
*
|
||
* Doesn't deal with:
|
||
* - re-freezing because this is called on a specific process and
|
||
* not by the freezer thread. If that changes, we'll have to teach it about
|
||
* refreezing a frozen process.
|
||
*
|
||
* - grouped/coalition freezing because we are hoping to deprecate this
|
||
* interface as it was used by user-space to freeze particular processes. But
|
||
* we have moved away from that approach to having the kernel choose the optimal
|
||
* candidates to be frozen.
|
||
*
|
||
* Returns ENOTSUP if the freezer isn't supported on this device. Otherwise
|
||
* returns EINVAL or the value returned by task_freeze().
|
||
*/
|
||
int
|
||
memorystatus_freeze_process_sync(proc_t p)
|
||
{
|
||
int ret = EINVAL;
|
||
boolean_t memorystatus_freeze_swap_low = FALSE;
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return ENOTSUP;
|
||
}
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
|
||
if (p == NULL) {
|
||
memorystatus_log_error("memorystatus_freeze_process_sync: Invalid process\n");
|
||
goto exit;
|
||
}
|
||
|
||
if (memorystatus_freeze_enabled == false) {
|
||
memorystatus_log_error("memorystatus_freeze_process_sync: Freezing is DISABLED\n");
|
||
goto exit;
|
||
}
|
||
|
||
if (!memorystatus_can_freeze(&memorystatus_freeze_swap_low)) {
|
||
memorystatus_log_info("memorystatus_freeze_process_sync: Low compressor and/or low swap space...skipping freeze\n");
|
||
goto exit;
|
||
}
|
||
|
||
memorystatus_freeze_update_throttle(&memorystatus_freeze_budget_pages_remaining);
|
||
if (!memorystatus_freeze_budget_pages_remaining) {
|
||
memorystatus_log_info("memorystatus_freeze_process_sync: exit with NO available budget\n");
|
||
goto exit;
|
||
}
|
||
|
||
proc_list_lock();
|
||
|
||
ret = memorystatus_freeze_process(p, NULL, NULL, NULL);
|
||
|
||
exit:
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
|
||
return ret;
|
||
}
|
||
|
||
proc_t
|
||
memorystatus_freezer_candidate_list_get_proc(
|
||
struct memorystatus_freezer_candidate_list *list,
|
||
size_t index,
|
||
uint64_t *pid_mismatch_counter)
|
||
{
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_OWNED);
|
||
if (list->mfcl_list == NULL || list->mfcl_length <= index) {
|
||
return NULL;
|
||
}
|
||
memorystatus_properties_freeze_entry_v1 *entry = &list->mfcl_list[index];
|
||
if (entry->pid == NO_PID) {
|
||
/* Entry has been removed. */
|
||
return NULL;
|
||
}
|
||
|
||
proc_t p = proc_find_locked(entry->pid);
|
||
if (p && strncmp(entry->proc_name, p->p_name, sizeof(proc_name_t)) == 0) {
|
||
/*
|
||
* We grab a reference when we are about to freeze the process. So drop
|
||
* the reference that proc_find_locked() grabbed for us.
|
||
* We also have the proc_list_lock so this process is stable.
|
||
*/
|
||
proc_rele(p);
|
||
return p;
|
||
} else {
|
||
if (p) {
|
||
/* pid rollover. */
|
||
proc_rele(p);
|
||
}
|
||
/*
|
||
* The proc has exited since we received this list.
|
||
* It may have re-launched with a new pid, so we go looking for it.
|
||
*/
|
||
unsigned int band = JETSAM_PRIORITY_IDLE;
|
||
p = memorystatus_get_first_proc_locked(&band, TRUE);
|
||
while (p != NULL && band <= memorystatus_freeze_max_candidate_band) {
|
||
if (strncmp(entry->proc_name, p->p_name, sizeof(proc_name_t)) == 0) {
|
||
if (pid_mismatch_counter != NULL) {
|
||
(*pid_mismatch_counter)++;
|
||
}
|
||
/* Stash the pid for faster lookup next time. */
|
||
entry->pid = proc_getpid(p);
|
||
return p;
|
||
}
|
||
p = memorystatus_get_next_proc_locked(&band, p, TRUE);
|
||
}
|
||
/* No match. */
|
||
return NULL;
|
||
}
|
||
}
|
||
|
||
static size_t
|
||
memorystatus_freeze_pid_list(pid_t *pid_list, unsigned int num_pids)
|
||
{
|
||
int ret = 0;
|
||
size_t num_frozen = 0;
|
||
while (num_pids > 0 &&
|
||
memorystatus_frozen_count < memorystatus_frozen_processes_max) {
|
||
pid_t pid = pid_list[--num_pids];
|
||
proc_t p = proc_find_locked(pid);
|
||
if (p) {
|
||
proc_rele(p);
|
||
ret = memorystatus_freeze_process(p, NULL, NULL, NULL);
|
||
if (ret != 0) {
|
||
break;
|
||
}
|
||
num_frozen++;
|
||
}
|
||
}
|
||
return num_frozen;
|
||
}
|
||
|
||
/*
|
||
* Attempt to freeze the best candidate process.
|
||
* Keep trying until we freeze something or run out of candidates.
|
||
* Returns the number of processes frozen (including coalition members).
|
||
*/
|
||
static size_t
|
||
memorystatus_freeze_top_process(void)
|
||
{
|
||
int freeze_ret;
|
||
size_t num_frozen = 0;
|
||
coalition_t coal = COALITION_NULL;
|
||
pid_t pid_list[MAX_XPC_SERVICE_PIDS];
|
||
unsigned int ntasks = 0;
|
||
struct memorystatus_freeze_list_iterator iterator;
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
|
||
bzero(&iterator, sizeof(struct memorystatus_freeze_list_iterator));
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE_SCAN) | DBG_FUNC_START, memorystatus_available_pages);
|
||
|
||
proc_list_lock();
|
||
while (true) {
|
||
proc_t p = memorystatus_freeze_pick_process(&iterator);
|
||
if (p == PROC_NULL) {
|
||
/* Nothing left to freeze */
|
||
break;
|
||
}
|
||
freeze_ret = memorystatus_freeze_process(p, &coal, pid_list, &ntasks);
|
||
if (freeze_ret == 0) {
|
||
num_frozen = 1;
|
||
/*
|
||
* We froze a process successfully.
|
||
* If it's a coalition head, freeze the coalition.
|
||
* Then we're done for now.
|
||
*/
|
||
if (coal != NULL) {
|
||
num_frozen += memorystatus_freeze_pid_list(pid_list, ntasks);
|
||
}
|
||
break;
|
||
} else {
|
||
if (vm_compressor_low_on_space() || vm_swap_low_on_space()) {
|
||
break;
|
||
}
|
||
/*
|
||
* Freeze failed but we're not out of space.
|
||
* Keep trying to find a good candidate,
|
||
* memorystatus_freeze_pick_process will not return this proc again until
|
||
* we reset the iterator.
|
||
*/
|
||
}
|
||
}
|
||
proc_list_unlock();
|
||
|
||
KDBG(MEMSTAT_CODE(BSD_MEMSTAT_FREEZE_SCAN) | DBG_FUNC_END, memorystatus_available_pages);
|
||
|
||
return num_frozen;
|
||
}
|
||
|
||
#if DEVELOPMENT || DEBUG
|
||
/* For testing memorystatus_freeze_top_process */
|
||
static int
|
||
sysctl_memorystatus_freeze_top_process SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int error, val, ret = 0;
|
||
size_t num_frozen;
|
||
/*
|
||
* Only freeze on write to prevent freezing during `sysctl -a`.
|
||
* The actual value written doesn't matter.
|
||
*/
|
||
error = sysctl_handle_int(oidp, &val, 0, req);
|
||
if (error || !req->newptr) {
|
||
return error;
|
||
}
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return ENOTSUP;
|
||
}
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
num_frozen = memorystatus_freeze_top_process();
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
|
||
if (num_frozen == 0) {
|
||
ret = ESRCH;
|
||
}
|
||
return ret;
|
||
}
|
||
SYSCTL_PROC(_vm, OID_AUTO, memorystatus_freeze_top_process, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED,
|
||
0, 0, &sysctl_memorystatus_freeze_top_process, "I", "");
|
||
#endif /* DEVELOPMENT || DEBUG */
|
||
|
||
static inline boolean_t
|
||
memorystatus_can_freeze_processes(void)
|
||
{
|
||
boolean_t ret;
|
||
|
||
proc_list_lock();
|
||
|
||
if (memorystatus_suspended_count) {
|
||
memorystatus_freeze_suspended_threshold = MIN(memorystatus_freeze_suspended_threshold, FREEZE_SUSPENDED_THRESHOLD_DEFAULT);
|
||
|
||
if ((memorystatus_suspended_count - memorystatus_frozen_count) > memorystatus_freeze_suspended_threshold) {
|
||
ret = TRUE;
|
||
} else {
|
||
ret = FALSE;
|
||
}
|
||
} else {
|
||
ret = FALSE;
|
||
}
|
||
|
||
proc_list_unlock();
|
||
|
||
return ret;
|
||
}
|
||
|
||
static boolean_t
|
||
memorystatus_can_freeze(boolean_t *memorystatus_freeze_swap_low)
|
||
{
|
||
boolean_t can_freeze = TRUE;
|
||
|
||
/* Only freeze if we're sufficiently low on memory; this holds off freeze right
|
||
* after boot, and is generally is a no-op once we've reached steady state. */
|
||
if (memorystatus_available_pages > memorystatus_freeze_threshold) {
|
||
return FALSE;
|
||
}
|
||
|
||
/* Check minimum suspended process threshold. */
|
||
if (!memorystatus_can_freeze_processes()) {
|
||
return FALSE;
|
||
}
|
||
assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
/*
|
||
* In-core compressor used for freezing WITHOUT on-disk swap support.
|
||
*/
|
||
if (vm_compressor_low_on_space()) {
|
||
if (*memorystatus_freeze_swap_low) {
|
||
*memorystatus_freeze_swap_low = TRUE;
|
||
}
|
||
|
||
can_freeze = FALSE;
|
||
} else {
|
||
if (*memorystatus_freeze_swap_low) {
|
||
*memorystatus_freeze_swap_low = FALSE;
|
||
}
|
||
|
||
can_freeze = TRUE;
|
||
}
|
||
} else {
|
||
/*
|
||
* Freezing WITH on-disk swap support.
|
||
*
|
||
* In-core compressor fronts the swap.
|
||
*/
|
||
if (vm_swap_low_on_space()) {
|
||
if (*memorystatus_freeze_swap_low) {
|
||
*memorystatus_freeze_swap_low = TRUE;
|
||
}
|
||
|
||
can_freeze = FALSE;
|
||
}
|
||
}
|
||
|
||
return can_freeze;
|
||
}
|
||
|
||
/*
|
||
* Demote the given frozen process.
|
||
* Caller must hold the proc_list_lock & it will be held on return.
|
||
*/
|
||
static void
|
||
memorystatus_demote_frozen_process(proc_t p, bool urgent_mode __unused)
|
||
{
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_OWNED);
|
||
|
||
/* We demote to IDLE unless someone has asserted a higher priority on this process. */
|
||
int maxpriority = JETSAM_PRIORITY_IDLE;
|
||
p->p_memstat_state &= ~P_MEMSTAT_USE_ELEVATED_INACTIVE_BAND;
|
||
memorystatus_invalidate_idle_demotion_locked(p, TRUE);
|
||
|
||
maxpriority = MAX(p->p_memstat_assertionpriority, maxpriority);
|
||
memorystatus_update_priority_locked(p, maxpriority, FALSE, FALSE);
|
||
#if DEVELOPMENT || DEBUG
|
||
memorystatus_log("memorystatus_demote_frozen_process(%s) pid %d [%s]\n",
|
||
(urgent_mode ? "urgent" : "normal"), (p ? proc_getpid(p) : -1), ((p && *p->p_name) ? p->p_name : "unknown"));
|
||
#endif /* DEVELOPMENT || DEBUG */
|
||
|
||
/*
|
||
* The freezer thread will consider this a normal app to be frozen
|
||
* because it is in the IDLE band. So we don't need the
|
||
* P_MEMSTAT_REFREEZE_ELIGIBLE state here. Also, if it gets resumed
|
||
* we'll correctly count it as eligible for re-freeze again.
|
||
*
|
||
* We don't drop the frozen count because this process still has
|
||
* state on disk. So there's a chance it gets resumed and then it
|
||
* should land in the higher jetsam band. For that it needs to
|
||
* remain marked frozen.
|
||
*/
|
||
if (memorystatus_freeze_proc_is_refreeze_eligible(p)) {
|
||
p->p_memstat_state &= ~P_MEMSTAT_REFREEZE_ELIGIBLE;
|
||
memorystatus_refreeze_eligible_count--;
|
||
}
|
||
}
|
||
|
||
static unsigned int
|
||
memorystatus_demote_frozen_processes_using_thaw_count(bool urgent_mode)
|
||
{
|
||
unsigned int band = (unsigned int) memorystatus_freeze_jetsam_band;
|
||
unsigned int demoted_proc_count = 0;
|
||
proc_t p = PROC_NULL, next_p = PROC_NULL;
|
||
proc_list_lock();
|
||
|
||
next_p = memorystatus_get_first_proc_locked(&band, FALSE);
|
||
while (next_p) {
|
||
p = next_p;
|
||
next_p = memorystatus_get_next_proc_locked(&band, p, FALSE);
|
||
|
||
if ((p->p_memstat_state & P_MEMSTAT_FROZEN) == FALSE) {
|
||
continue;
|
||
}
|
||
|
||
if (p->p_memstat_state & P_MEMSTAT_LOCKED) {
|
||
continue;
|
||
}
|
||
|
||
if (urgent_mode) {
|
||
if (!memorystatus_freeze_proc_is_refreeze_eligible(p)) {
|
||
/*
|
||
* This process hasn't been thawed recently and so most of
|
||
* its state sits on NAND and so we skip it -- jetsamming it
|
||
* won't help with memory pressure.
|
||
*/
|
||
continue;
|
||
}
|
||
} else {
|
||
if (p->p_memstat_thaw_count >= memorystatus_thaw_count_demotion_threshold) {
|
||
/*
|
||
* This process has met / exceeded our thaw count demotion threshold
|
||
* and so we let it live in the higher bands.
|
||
*/
|
||
continue;
|
||
}
|
||
}
|
||
|
||
memorystatus_demote_frozen_process(p, urgent_mode);
|
||
demoted_proc_count++;
|
||
if ((urgent_mode) || (demoted_proc_count == memorystatus_max_frozen_demotions_daily)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
proc_list_unlock();
|
||
return demoted_proc_count;
|
||
}
|
||
|
||
static unsigned int
|
||
memorystatus_demote_frozen_processes_using_demote_list(bool urgent_mode)
|
||
{
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
|
||
assert(memorystatus_freezer_use_demotion_list);
|
||
unsigned int demoted_proc_count = 0;
|
||
|
||
proc_list_lock();
|
||
for (size_t i = 0; i < memorystatus_global_demote_list.mfcl_length; i++) {
|
||
proc_t p = memorystatus_freezer_candidate_list_get_proc(
|
||
&memorystatus_global_demote_list,
|
||
i,
|
||
&memorystatus_freezer_stats.mfs_demote_pid_mismatches);
|
||
if (p != NULL && memorystatus_freeze_proc_is_refreeze_eligible(p)) {
|
||
memorystatus_demote_frozen_process(p, urgent_mode);
|
||
/* Remove this entry now that it's been demoted. */
|
||
memorystatus_global_demote_list.mfcl_list[i].pid = NO_PID;
|
||
demoted_proc_count++;
|
||
/*
|
||
* We only demote one proc at a time in this mode.
|
||
* This gives jetsam a chance to kill the recently demoted processes.
|
||
*/
|
||
break;
|
||
}
|
||
}
|
||
|
||
proc_list_unlock();
|
||
return demoted_proc_count;
|
||
}
|
||
|
||
/*
|
||
* This function evaluates if the currently frozen processes deserve
|
||
* to stay in the higher jetsam band. There are 2 modes:
|
||
* - 'force one == TRUE': (urgent mode)
|
||
* We are out of budget and can't refreeze a process. The process's
|
||
* state, if it was resumed, will stay in compressed memory. If we let it
|
||
* remain up in the higher frozen jetsam band, it'll put a lot of pressure on
|
||
* the lower bands. So we force-demote the least-recently-used-and-thawed
|
||
* process.
|
||
*
|
||
* - 'force_one == FALSE': (normal mode)
|
||
* If the # of thaws of a process is below our threshold, then we
|
||
* will demote that process into the IDLE band.
|
||
* We don't immediately kill the process here because it already has
|
||
* state on disk and so it might be worth giving it another shot at
|
||
* getting thawed/resumed and used.
|
||
*/
|
||
static void
|
||
memorystatus_demote_frozen_processes(bool urgent_mode)
|
||
{
|
||
unsigned int demoted_proc_count = 0;
|
||
|
||
if (memorystatus_freeze_enabled == false) {
|
||
/*
|
||
* Freeze has been disabled likely to
|
||
* reclaim swap space. So don't change
|
||
* any state on the frozen processes.
|
||
*/
|
||
return;
|
||
}
|
||
|
||
/*
|
||
* We have two demotion policies which can be toggled by userspace.
|
||
* In non-urgent mode, the ordered list policy will
|
||
* choose a demotion candidate using the list provided by dasd.
|
||
* The thaw count policy will demote the oldest process that hasn't been
|
||
* thawed more than memorystatus_thaw_count_demotion_threshold times.
|
||
*
|
||
* If urgent_mode is set, both policies will only consider demoting
|
||
* processes that are re-freeze eligible. But the ordering is different.
|
||
* The ordered list policy will scan in the order given by dasd.
|
||
* The thaw count policy will scan through the frozen band.
|
||
*/
|
||
if (memorystatus_freezer_use_demotion_list) {
|
||
demoted_proc_count += memorystatus_demote_frozen_processes_using_demote_list(urgent_mode);
|
||
|
||
if (demoted_proc_count == 0 && urgent_mode) {
|
||
/*
|
||
* We're out of budget and the demotion list doesn't contain any valid
|
||
* candidates. We still need to demote something. Fall back to scanning
|
||
* the frozen band.
|
||
*/
|
||
memorystatus_demote_frozen_processes_using_thaw_count(true);
|
||
}
|
||
} else {
|
||
demoted_proc_count += memorystatus_demote_frozen_processes_using_thaw_count(urgent_mode);
|
||
}
|
||
}
|
||
|
||
/*
|
||
* Calculate a new freezer budget.
|
||
* @param time_since_last_interval_expired_sec How long has it been (in seconds) since the previous interval expired.
|
||
* @param burst_multiple The burst_multiple for the new period
|
||
* @param interval_duration_min How many minutes will the new interval be?
|
||
* @param rollover The amount to rollover from the previous budget.
|
||
*
|
||
* @return A budget for the new interval.
|
||
*/
|
||
static uint32_t
|
||
memorystatus_freeze_calculate_new_budget(
|
||
unsigned int time_since_last_interval_expired_sec,
|
||
unsigned int burst_multiple,
|
||
unsigned int interval_duration_min,
|
||
uint32_t rollover)
|
||
{
|
||
uint64_t freeze_daily_budget = 0, freeze_daily_budget_mb = 0, daily_budget_pageouts = 0, budget_missed = 0, freeze_daily_pageouts_max = 0, new_budget = 0;
|
||
const static unsigned int kNumSecondsInDay = 60 * 60 * 24;
|
||
/* Precision factor for days_missed. 2 decimal points. */
|
||
const static unsigned int kFixedPointFactor = 100;
|
||
unsigned int days_missed;
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return 0;
|
||
}
|
||
if (memorystatus_swap_all_apps) {
|
||
/*
|
||
* We effectively have an unlimited budget when app swap is enabled.
|
||
*/
|
||
memorystatus_freeze_daily_mb_max = UINT32_MAX;
|
||
return UINT32_MAX;
|
||
}
|
||
|
||
/* Get the daily budget from the storage layer */
|
||
if (vm_swap_max_budget(&freeze_daily_budget)) {
|
||
freeze_daily_budget_mb = freeze_daily_budget / (1024 * 1024);
|
||
assert(freeze_daily_budget_mb <= UINT32_MAX);
|
||
memorystatus_freeze_daily_mb_max = (unsigned int) freeze_daily_budget_mb;
|
||
memorystatus_log_info("memorystatus: memorystatus_freeze_daily_mb_max set to %dMB\n", memorystatus_freeze_daily_mb_max);
|
||
}
|
||
/* Calculate the daily pageout budget */
|
||
freeze_daily_pageouts_max = memorystatus_freeze_daily_mb_max * (1024 * 1024 / PAGE_SIZE);
|
||
/* Multiply by memorystatus_freeze_budget_multiplier */
|
||
freeze_daily_pageouts_max = ((kFixedPointFactor * memorystatus_freeze_budget_multiplier / 100) * freeze_daily_pageouts_max) / kFixedPointFactor;
|
||
|
||
daily_budget_pageouts = (burst_multiple * (((uint64_t) interval_duration_min * freeze_daily_pageouts_max) / (kNumSecondsInDay / 60)));
|
||
|
||
/*
|
||
* Add additional budget for time since the interval expired.
|
||
* For example, if the interval expired n days ago, we should get an additional n days
|
||
* of budget since we didn't use any budget during those n days.
|
||
*/
|
||
days_missed = time_since_last_interval_expired_sec * kFixedPointFactor / kNumSecondsInDay;
|
||
budget_missed = days_missed * freeze_daily_pageouts_max / kFixedPointFactor;
|
||
new_budget = rollover + daily_budget_pageouts + budget_missed;
|
||
return (uint32_t) MIN(new_budget, UINT32_MAX);
|
||
}
|
||
|
||
/*
|
||
* Mark all non frozen, freezer-eligible processes as skipped for the given reason.
|
||
* Used when we hit some system freeze limit and know that we won't be considering remaining processes.
|
||
* If you're using this for a new reason, make sure to add it to memorystatus_freeze_init_proc so that
|
||
* it gets set for new processes.
|
||
* NB: These processes will retain this skip reason until they are reconsidered by memorystatus_is_process_eligible_for_freeze.
|
||
*/
|
||
static void
|
||
memorystatus_freeze_mark_eligible_processes_with_skip_reason(memorystatus_freeze_skip_reason_t reason, bool locked)
|
||
{
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, locked ? LCK_MTX_ASSERT_OWNED : LCK_MTX_ASSERT_NOTOWNED);
|
||
unsigned int band = JETSAM_PRIORITY_IDLE;
|
||
proc_t p;
|
||
|
||
if (!locked) {
|
||
proc_list_lock();
|
||
}
|
||
p = memorystatus_get_first_proc_locked(&band, FALSE);
|
||
while (p) {
|
||
assert(p->p_memstat_effectivepriority == (int32_t) band);
|
||
if (!(p->p_memstat_state & P_MEMSTAT_FROZEN) && memorystatus_is_process_eligible_for_freeze(p)) {
|
||
assert(p->p_memstat_freeze_skip_reason == kMemorystatusFreezeSkipReasonNone);
|
||
p->p_memstat_freeze_skip_reason = (uint8_t) reason;
|
||
}
|
||
p = memorystatus_get_next_proc_locked(&band, p, FALSE);
|
||
}
|
||
if (!locked) {
|
||
proc_list_unlock();
|
||
}
|
||
}
|
||
|
||
/*
|
||
* Called after we fail to freeze a process.
|
||
* Logs the failure, marks the process with the failure reason, and updates freezer stats.
|
||
*/
|
||
static void
|
||
memorystatus_freeze_handle_error(
|
||
proc_t p,
|
||
const freezer_error_code_t freezer_error_code,
|
||
bool was_refreeze,
|
||
pid_t pid,
|
||
const coalition_t coalition,
|
||
const char* log_prefix)
|
||
{
|
||
const char *reason;
|
||
memorystatus_freeze_skip_reason_t skip_reason;
|
||
|
||
switch (freezer_error_code) {
|
||
case FREEZER_ERROR_EXCESS_SHARED_MEMORY:
|
||
memorystatus_freezer_stats.mfs_error_excess_shared_memory_count++;
|
||
reason = "too much shared memory";
|
||
skip_reason = kMemorystatusFreezeSkipReasonExcessSharedMemory;
|
||
break;
|
||
case FREEZER_ERROR_LOW_PRIVATE_SHARED_RATIO:
|
||
memorystatus_freezer_stats.mfs_error_low_private_shared_ratio_count++;
|
||
reason = "private-shared pages ratio";
|
||
skip_reason = kMemorystatusFreezeSkipReasonLowPrivateSharedRatio;
|
||
break;
|
||
case FREEZER_ERROR_NO_COMPRESSOR_SPACE:
|
||
memorystatus_freezer_stats.mfs_error_no_compressor_space_count++;
|
||
reason = "no compressor space";
|
||
skip_reason = kMemorystatusFreezeSkipReasonNoCompressorSpace;
|
||
break;
|
||
case FREEZER_ERROR_NO_SWAP_SPACE:
|
||
memorystatus_freezer_stats.mfs_error_no_swap_space_count++;
|
||
reason = "no swap space";
|
||
skip_reason = kMemorystatusFreezeSkipReasonNoSwapSpace;
|
||
break;
|
||
case FREEZER_ERROR_NO_SLOTS:
|
||
memorystatus_freezer_stats.mfs_skipped_full_count++;
|
||
reason = "no slots";
|
||
skip_reason = kMemorystatusFreezeSkipReasonOutOfSlots;
|
||
break;
|
||
default:
|
||
reason = "unknown error";
|
||
skip_reason = kMemorystatusFreezeSkipReasonOther;
|
||
}
|
||
|
||
p->p_memstat_freeze_skip_reason = (uint8_t) skip_reason;
|
||
|
||
memorystatus_log("%s: %sfreezing (%s) pid %d [%s]...skipped (%s)\n",
|
||
log_prefix, was_refreeze ? "re" : "",
|
||
(coalition == NULL ? "general" : "coalition-driven"), pid,
|
||
((p && *p->p_name) ? p->p_name : "unknown"), reason);
|
||
}
|
||
|
||
/*
|
||
* Start a new normal throttle interval with the given budget.
|
||
* Caller must hold the freezer mutex
|
||
*/
|
||
static void
|
||
memorystatus_freeze_start_normal_throttle_interval(uint32_t new_budget, mach_timespec_t start_ts)
|
||
{
|
||
unsigned int band;
|
||
proc_t p, next_p;
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
|
||
|
||
normal_throttle_window->max_pageouts = new_budget;
|
||
normal_throttle_window->ts.tv_sec = normal_throttle_window->mins * 60;
|
||
normal_throttle_window->ts.tv_nsec = 0;
|
||
ADD_MACH_TIMESPEC(&normal_throttle_window->ts, &start_ts);
|
||
/* Since we update the throttle stats pre-freeze, adjust for overshoot here */
|
||
if (normal_throttle_window->pageouts > normal_throttle_window->max_pageouts) {
|
||
normal_throttle_window->pageouts -= normal_throttle_window->max_pageouts;
|
||
} else {
|
||
normal_throttle_window->pageouts = 0;
|
||
}
|
||
/* Ensure the normal window is now active. */
|
||
memorystatus_freeze_degradation = FALSE;
|
||
|
||
/*
|
||
* Reset interval statistics.
|
||
*/
|
||
memorystatus_freezer_stats.mfs_shared_pages_skipped = 0;
|
||
memorystatus_freezer_stats.mfs_process_considered_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_below_min_pages_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_excess_shared_memory_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_low_private_shared_ratio_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_no_compressor_space_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_no_swap_space_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_low_probability_of_use_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_elevated_count = 0;
|
||
memorystatus_freezer_stats.mfs_error_other_count = 0;
|
||
memorystatus_freezer_stats.mfs_refreeze_count = 0;
|
||
memorystatus_freezer_stats.mfs_bytes_refrozen = 0;
|
||
memorystatus_freezer_stats.mfs_below_threshold_count = 0;
|
||
memorystatus_freezer_stats.mfs_skipped_full_count = 0;
|
||
memorystatus_freezer_stats.mfs_skipped_shared_mb_high_count = 0;
|
||
memorystatus_freezer_stats.mfs_budget_exhaustion_duration_remaining = 0;
|
||
memorystatus_thaw_count = 0;
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_thawed, 0, release);
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_thawed_webcontent, 0, release);
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_thawed_fg, 0, release);
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_thawed_fg_xpc_service, 0, release);
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_frozen, memorystatus_frozen_count, release);
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_frozen_webcontent, memorystatus_frozen_count_webcontent, release);
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_frozen_xpc_service, memorystatus_frozen_count_xpc_service, release);
|
||
os_atomic_store(&memorystatus_freezer_stats.mfs_processes_fg_resumed, 0, release);
|
||
os_atomic_inc(&memorystatus_freeze_current_interval, release);
|
||
|
||
/* Clear the focal thaw bit */
|
||
proc_list_lock();
|
||
band = JETSAM_PRIORITY_IDLE;
|
||
p = PROC_NULL;
|
||
next_p = PROC_NULL;
|
||
|
||
next_p = memorystatus_get_first_proc_locked(&band, TRUE);
|
||
while (next_p) {
|
||
p = next_p;
|
||
next_p = memorystatus_get_next_proc_locked(&band, p, TRUE);
|
||
|
||
if (p->p_memstat_effectivepriority > JETSAM_PRIORITY_FOREGROUND) {
|
||
break;
|
||
}
|
||
p->p_memstat_state &= ~P_MEMSTAT_FROZEN_FOCAL_THAW;
|
||
}
|
||
proc_list_unlock();
|
||
|
||
schedule_interval_reset(freeze_interval_reset_thread_call, normal_throttle_window);
|
||
}
|
||
|
||
#if DEVELOPMENT || DEBUG
|
||
|
||
static int
|
||
sysctl_memorystatus_freeze_calculate_new_budget SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(arg1, arg2)
|
||
int error = 0;
|
||
unsigned int time_since_last_interval_expired_sec = 0;
|
||
unsigned int new_budget;
|
||
|
||
error = sysctl_handle_int(oidp, &time_since_last_interval_expired_sec, 0, req);
|
||
if (error || !req->newptr) {
|
||
return error;
|
||
}
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return ENOTSUP;
|
||
}
|
||
new_budget = memorystatus_freeze_calculate_new_budget(time_since_last_interval_expired_sec, 1, NORMAL_WINDOW_MINS, 0);
|
||
return copyout(&new_budget, req->oldptr, MIN(sizeof(req->oldlen), sizeof(new_budget)));
|
||
}
|
||
|
||
SYSCTL_PROC(_vm, OID_AUTO, memorystatus_freeze_calculate_new_budget, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED,
|
||
0, 0, &sysctl_memorystatus_freeze_calculate_new_budget, "I", "");
|
||
|
||
#endif /* DEVELOPMENT || DEBUG */
|
||
|
||
/*
|
||
* Called when we first run out of budget in an interval.
|
||
* Marks idle processes as not frozen due to lack of budget.
|
||
* NB: It might be worth having a CA event here.
|
||
*/
|
||
static void
|
||
memorystatus_freeze_out_of_budget(const struct throttle_interval_t *interval)
|
||
{
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
|
||
|
||
mach_timespec_t time_left = {0, 0};
|
||
mach_timespec_t now_ts;
|
||
clock_sec_t sec;
|
||
clock_nsec_t nsec;
|
||
|
||
time_left.tv_sec = interval->ts.tv_sec;
|
||
time_left.tv_nsec = 0;
|
||
clock_get_system_nanotime(&sec, &nsec);
|
||
now_ts.tv_sec = (unsigned int)(MIN(sec, UINT32_MAX));
|
||
now_ts.tv_nsec = nsec;
|
||
|
||
SUB_MACH_TIMESPEC(&time_left, &now_ts);
|
||
memorystatus_freezer_stats.mfs_budget_exhaustion_duration_remaining = time_left.tv_sec;
|
||
memorystatus_log(
|
||
"memorystatus_freeze: Out of NAND write budget with %u minutes left in the current freezer interval. %u procs are frozen.\n",
|
||
time_left.tv_sec / 60, memorystatus_frozen_count);
|
||
|
||
memorystatus_freeze_mark_eligible_processes_with_skip_reason(kMemorystatusFreezeSkipReasonOutOfBudget, false);
|
||
}
|
||
|
||
/*
|
||
* Called when we cross over the threshold of maximum frozen processes allowed.
|
||
* Marks remaining idle processes as not frozen due to lack of slots.
|
||
*/
|
||
static void
|
||
memorystatus_freeze_out_of_slots(void)
|
||
{
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_OWNED);
|
||
assert(memorystatus_frozen_count == memorystatus_frozen_processes_max);
|
||
|
||
memorystatus_log(
|
||
"memorystatus_freeze: Out of slots in the freezer. %u procs are frozen.\n",
|
||
memorystatus_frozen_count);
|
||
|
||
memorystatus_freeze_mark_eligible_processes_with_skip_reason(kMemorystatusFreezeSkipReasonOutOfSlots, true);
|
||
}
|
||
|
||
/*
|
||
* This function will do 4 things:
|
||
*
|
||
* 1) check to see if we are currently in a degraded freezer mode, and if so:
|
||
* - check to see if our window has expired and we should exit this mode, OR,
|
||
* - return a budget based on the degraded throttle window's max. pageouts vs current pageouts.
|
||
*
|
||
* 2) check to see if we are in a NEW normal window and update the normal throttle window's params.
|
||
*
|
||
* 3) check what the current normal window allows for a budget.
|
||
*
|
||
* 4) calculate the current rate of pageouts for DEGRADED_WINDOW_MINS duration. If that rate is below
|
||
* what we would normally expect, then we are running low on our daily budget and need to enter
|
||
* degraded perf. mode.
|
||
*
|
||
* Caller must hold the freezer mutex
|
||
* Caller must not hold the proc_list lock
|
||
*/
|
||
|
||
static void
|
||
memorystatus_freeze_update_throttle(uint64_t *budget_pages_allowed)
|
||
{
|
||
clock_sec_t sec;
|
||
clock_nsec_t nsec;
|
||
mach_timespec_t now_ts;
|
||
LCK_MTX_ASSERT(&freezer_mutex, LCK_MTX_ASSERT_OWNED);
|
||
LCK_MTX_ASSERT(&proc_list_mlock, LCK_MTX_ASSERT_NOTOWNED);
|
||
|
||
unsigned int freeze_daily_pageouts_max = 0;
|
||
bool started_with_budget = (*budget_pages_allowed > 0);
|
||
|
||
#if DEVELOPMENT || DEBUG
|
||
if (!memorystatus_freeze_throttle_enabled) {
|
||
/*
|
||
* No throttling...we can use the full budget everytime.
|
||
*/
|
||
*budget_pages_allowed = UINT64_MAX;
|
||
return;
|
||
}
|
||
#endif
|
||
|
||
clock_get_system_nanotime(&sec, &nsec);
|
||
now_ts.tv_sec = (unsigned int)(MIN(sec, UINT32_MAX));
|
||
now_ts.tv_nsec = nsec;
|
||
|
||
struct throttle_interval_t *interval = NULL;
|
||
|
||
if (memorystatus_freeze_degradation == TRUE) {
|
||
interval = degraded_throttle_window;
|
||
|
||
if (CMP_MACH_TIMESPEC(&now_ts, &interval->ts) >= 0) {
|
||
interval->pageouts = 0;
|
||
interval->max_pageouts = 0;
|
||
} else {
|
||
*budget_pages_allowed = interval->max_pageouts - interval->pageouts;
|
||
}
|
||
}
|
||
|
||
interval = normal_throttle_window;
|
||
|
||
/*
|
||
* Current throttle window.
|
||
* Deny freezing if we have no budget left.
|
||
* Try graceful degradation if we are within 25% of:
|
||
* - the daily budget, and
|
||
* - the current budget left is below our normal budget expectations.
|
||
*/
|
||
|
||
if (memorystatus_freeze_degradation == FALSE) {
|
||
if (interval->pageouts >= interval->max_pageouts) {
|
||
*budget_pages_allowed = 0;
|
||
if (started_with_budget) {
|
||
memorystatus_freeze_out_of_budget(interval);
|
||
}
|
||
} else {
|
||
int budget_left = interval->max_pageouts - interval->pageouts;
|
||
int budget_threshold = (freeze_daily_pageouts_max * FREEZE_DEGRADATION_BUDGET_THRESHOLD) / 100;
|
||
|
||
mach_timespec_t time_left = {0, 0};
|
||
|
||
time_left.tv_sec = interval->ts.tv_sec;
|
||
time_left.tv_nsec = 0;
|
||
|
||
SUB_MACH_TIMESPEC(&time_left, &now_ts);
|
||
|
||
if (budget_left <= budget_threshold) {
|
||
/*
|
||
* For the current normal window, calculate how much we would pageout in a DEGRADED_WINDOW_MINS duration.
|
||
* And also calculate what we would pageout for the same DEGRADED_WINDOW_MINS duration if we had the full
|
||
* daily pageout budget.
|
||
*/
|
||
|
||
unsigned int current_budget_rate_allowed = ((budget_left / time_left.tv_sec) / 60) * DEGRADED_WINDOW_MINS;
|
||
unsigned int normal_budget_rate_allowed = (freeze_daily_pageouts_max / NORMAL_WINDOW_MINS) * DEGRADED_WINDOW_MINS;
|
||
|
||
/*
|
||
* The current rate of pageouts is below what we would expect for
|
||
* the normal rate i.e. we have below normal budget left and so...
|
||
*/
|
||
|
||
if (current_budget_rate_allowed < normal_budget_rate_allowed) {
|
||
memorystatus_freeze_degradation = TRUE;
|
||
degraded_throttle_window->max_pageouts = current_budget_rate_allowed;
|
||
degraded_throttle_window->pageouts = 0;
|
||
|
||
/*
|
||
* Switch over to the degraded throttle window so the budget
|
||
* doled out is based on that window.
|
||
*/
|
||
interval = degraded_throttle_window;
|
||
}
|
||
}
|
||
|
||
*budget_pages_allowed = interval->max_pageouts - interval->pageouts;
|
||
}
|
||
}
|
||
|
||
memorystatus_log_debug(
|
||
"memorystatus_freeze_update_throttle_interval: throttle updated - %d frozen (%d max) within %dm; %dm remaining\n",
|
||
interval->pageouts, interval->max_pageouts, interval->mins, (interval->ts.tv_sec - now_ts.tv_sec) / 60);
|
||
}
|
||
|
||
SYSCTL_UINT(_kern, OID_AUTO, memorystatus_freeze_apps_idle_delay_multiplier, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_freeze_apps_idle_delay_multiplier, 0, "");
|
||
|
||
bool memorystatus_freeze_thread_init = false;
|
||
static void
|
||
memorystatus_freeze_thread(void *param __unused, wait_result_t wr __unused)
|
||
{
|
||
static boolean_t memorystatus_freeze_swap_low = FALSE;
|
||
size_t max_to_freeze = 0, num_frozen = 0, num_frozen_this_iteration = 0;
|
||
|
||
if (!memorystatus_freeze_thread_init) {
|
||
#if CONFIG_THREAD_GROUPS
|
||
thread_group_vm_add();
|
||
#endif
|
||
memorystatus_freeze_thread_init = true;
|
||
}
|
||
|
||
max_to_freeze = memorystatus_pick_freeze_count_for_wakeup();
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
if (memorystatus_freeze_enabled) {
|
||
if (memorystatus_freezer_use_demotion_list && memorystatus_refreeze_eligible_count > 0) {
|
||
memorystatus_demote_frozen_processes(false); /* Normal mode. Consider demoting thawed processes. */
|
||
}
|
||
while (num_frozen < max_to_freeze &&
|
||
memorystatus_can_freeze(&memorystatus_freeze_swap_low) &&
|
||
((memorystatus_frozen_count < memorystatus_frozen_processes_max) ||
|
||
(memorystatus_refreeze_eligible_count >= memorystatus_min_thaw_refreeze_threshold))) {
|
||
/* Only freeze if we've not exceeded our pageout budgets.*/
|
||
memorystatus_freeze_update_throttle(&memorystatus_freeze_budget_pages_remaining);
|
||
|
||
if (memorystatus_freeze_budget_pages_remaining) {
|
||
num_frozen_this_iteration = memorystatus_freeze_top_process();
|
||
if (num_frozen_this_iteration == 0) {
|
||
/* Nothing left to freeze. */
|
||
break;
|
||
}
|
||
num_frozen += num_frozen_this_iteration;
|
||
} else {
|
||
memorystatus_demote_frozen_processes(true); /* urgent mode..force one demotion */
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
/*
|
||
* Give applications currently in the aging band a chance to age out into the idle band before
|
||
* running the freezer again.
|
||
*/
|
||
if (memorystatus_freeze_dynamic_thread_delay_enabled) {
|
||
if ((num_frozen > 0) || (memorystatus_frozen_count == 0)) {
|
||
memorystatus_freeze_apps_idle_delay_multiplier = FREEZE_APPS_IDLE_DELAY_MULTIPLIER_FAST;
|
||
} else {
|
||
memorystatus_freeze_apps_idle_delay_multiplier = FREEZE_APPS_IDLE_DELAY_MULTIPLIER_SLOW;
|
||
}
|
||
}
|
||
memorystatus_freezer_thread_next_run_ts = mach_absolute_time() + (memorystatus_apps_idle_delay_time * memorystatus_freeze_apps_idle_delay_multiplier);
|
||
|
||
assert_wait((event_t) &memorystatus_freeze_wakeup, THREAD_UNINT);
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
|
||
thread_block((thread_continue_t) memorystatus_freeze_thread);
|
||
}
|
||
|
||
int
|
||
memorystatus_get_process_is_freezable(pid_t pid, int *is_freezable)
|
||
{
|
||
proc_t p = PROC_NULL;
|
||
|
||
if (pid == 0) {
|
||
return EINVAL;
|
||
}
|
||
|
||
p = proc_find(pid);
|
||
if (!p) {
|
||
return ESRCH;
|
||
}
|
||
|
||
/*
|
||
* Only allow this on the current proc for now.
|
||
* We can check for privileges and allow targeting another process in the future.
|
||
*/
|
||
if (p != current_proc()) {
|
||
proc_rele(p);
|
||
return EPERM;
|
||
}
|
||
|
||
proc_list_lock();
|
||
*is_freezable = ((p->p_memstat_state & P_MEMSTAT_FREEZE_DISABLED) ? 0 : 1);
|
||
proc_rele(p);
|
||
proc_list_unlock();
|
||
|
||
return 0;
|
||
}
|
||
|
||
errno_t
|
||
memorystatus_get_process_is_frozen(pid_t pid, int *is_frozen)
|
||
{
|
||
proc_t p = PROC_NULL;
|
||
|
||
if (pid == 0) {
|
||
return EINVAL;
|
||
}
|
||
|
||
/*
|
||
* Only allow this on the current proc for now.
|
||
* We can check for privileges and allow targeting another process in the future.
|
||
*/
|
||
p = current_proc();
|
||
if (proc_getpid(p) != pid) {
|
||
return EPERM;
|
||
}
|
||
|
||
proc_list_lock();
|
||
*is_frozen = (p->p_memstat_state & P_MEMSTAT_FROZEN) != 0;
|
||
proc_list_unlock();
|
||
|
||
return 0;
|
||
}
|
||
|
||
int
|
||
memorystatus_set_process_is_freezable(pid_t pid, boolean_t is_freezable)
|
||
{
|
||
proc_t p = PROC_NULL;
|
||
|
||
if (pid == 0) {
|
||
return EINVAL;
|
||
}
|
||
|
||
/*
|
||
* To enable freezable status, you need to be root or an entitlement.
|
||
*/
|
||
if (is_freezable &&
|
||
!kauth_cred_issuser(kauth_cred_get()) &&
|
||
!IOCurrentTaskHasEntitlement(MEMORYSTATUS_ENTITLEMENT)) {
|
||
return EPERM;
|
||
}
|
||
|
||
p = proc_find(pid);
|
||
if (!p) {
|
||
return ESRCH;
|
||
}
|
||
|
||
/*
|
||
* A process can change its own status. A coalition leader can
|
||
* change the status of coalition members.
|
||
* An entitled process (or root) can change anyone's status.
|
||
*/
|
||
if (p != current_proc() &&
|
||
!kauth_cred_issuser(kauth_cred_get()) &&
|
||
!IOCurrentTaskHasEntitlement(MEMORYSTATUS_ENTITLEMENT)) {
|
||
coalition_t coal = task_get_coalition(proc_task(p), COALITION_TYPE_JETSAM);
|
||
if (!coalition_is_leader(proc_task(current_proc()), coal)) {
|
||
proc_rele(p);
|
||
return EPERM;
|
||
}
|
||
}
|
||
|
||
proc_list_lock();
|
||
if (is_freezable == FALSE) {
|
||
/* Freeze preference set to FALSE. Set the P_MEMSTAT_FREEZE_DISABLED bit. */
|
||
p->p_memstat_state |= P_MEMSTAT_FREEZE_DISABLED;
|
||
memorystatus_log_info("memorystatus_set_process_is_freezable: disabling freeze for pid %d [%s]\n",
|
||
proc_getpid(p), (*p->p_name ? p->p_name : "unknown"));
|
||
} else {
|
||
p->p_memstat_state &= ~P_MEMSTAT_FREEZE_DISABLED;
|
||
memorystatus_log_info("memorystatus_set_process_is_freezable: enabling freeze for pid %d [%s]\n",
|
||
proc_getpid(p), (*p->p_name ? p->p_name : "unknown"));
|
||
}
|
||
proc_rele(p);
|
||
proc_list_unlock();
|
||
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* Called when process is created before it is added to a memorystatus bucket.
|
||
*/
|
||
void
|
||
memorystatus_freeze_init_proc(proc_t p)
|
||
{
|
||
/* NB: Process is not on the memorystatus lists yet so it's safe to modify the skip reason without the freezer mutex. */
|
||
if (memorystatus_freeze_budget_pages_remaining == 0) {
|
||
p->p_memstat_freeze_skip_reason = kMemorystatusFreezeSkipReasonOutOfBudget;
|
||
} else if ((memorystatus_frozen_count >= memorystatus_frozen_processes_max)) {
|
||
p->p_memstat_freeze_skip_reason = kMemorystatusFreezeSkipReasonOutOfSlots;
|
||
} else {
|
||
p->p_memstat_freeze_skip_reason = kMemorystatusFreezeSkipReasonNone;
|
||
}
|
||
}
|
||
|
||
|
||
static int
|
||
sysctl_memorystatus_do_fastwake_warmup_all SYSCTL_HANDLER_ARGS
|
||
{
|
||
#pragma unused(oidp, arg1, arg2)
|
||
|
||
if (!req->newptr) {
|
||
return EINVAL;
|
||
}
|
||
|
||
/* Need to be root or have entitlement */
|
||
if (!kauth_cred_issuser(kauth_cred_get()) && !IOCurrentTaskHasEntitlement( MEMORYSTATUS_ENTITLEMENT)) {
|
||
return EPERM;
|
||
}
|
||
|
||
if (memorystatus_freeze_enabled == false) {
|
||
return ENOTSUP;
|
||
}
|
||
|
||
if (!VM_CONFIG_FREEZER_SWAP_IS_ACTIVE) {
|
||
return ENOTSUP;
|
||
}
|
||
|
||
do_fastwake_warmup_all();
|
||
|
||
return 0;
|
||
}
|
||
|
||
SYSCTL_PROC(_kern, OID_AUTO, memorystatus_do_fastwake_warmup_all, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
|
||
0, 0, &sysctl_memorystatus_do_fastwake_warmup_all, "I", "");
|
||
|
||
/*
|
||
* Takes in a candidate list from the user_addr, validates it, and copies it into the list pointer.
|
||
* Takes ownership over the original value of list.
|
||
* Assumes that list is protected by the freezer_mutex.
|
||
* The caller should not hold any locks.
|
||
*/
|
||
static errno_t
|
||
set_freezer_candidate_list(user_addr_t buffer, size_t buffer_size, struct memorystatus_freezer_candidate_list *list)
|
||
{
|
||
errno_t error = 0;
|
||
memorystatus_properties_freeze_entry_v1 *entries = NULL, *tmp_entries = NULL;
|
||
size_t entry_count = 0, entries_size = 0, tmp_size = 0;
|
||
|
||
/* Validate the user provided list. */
|
||
if ((buffer == USER_ADDR_NULL) || (buffer_size == 0)) {
|
||
memorystatus_log_error("memorystatus_cmd_grp_set_freeze_priority: NULL or empty list\n");
|
||
return EINVAL;
|
||
}
|
||
|
||
if (buffer_size % sizeof(memorystatus_properties_freeze_entry_v1) != 0) {
|
||
memorystatus_log_error(
|
||
"memorystatus_cmd_grp_set_freeze_priority: Invalid list length (caller might have comiled agsinst invalid headers.)\n");
|
||
return EINVAL;
|
||
}
|
||
|
||
entry_count = buffer_size / sizeof(memorystatus_properties_freeze_entry_v1);
|
||
entries_size = buffer_size;
|
||
entries = kalloc_data(buffer_size, Z_WAITOK | Z_ZERO);
|
||
if (entries == NULL) {
|
||
return ENOMEM;
|
||
}
|
||
|
||
error = copyin(buffer, entries, buffer_size);
|
||
if (error != 0) {
|
||
goto out;
|
||
}
|
||
|
||
#if MACH_ASSERT
|
||
for (size_t i = 0; i < entry_count; i++) {
|
||
memorystatus_properties_freeze_entry_v1 *entry = &entries[i];
|
||
if (entry->version != 1) {
|
||
memorystatus_log_error("memorystatus_cmd_grp_set_freeze_priority: Invalid entry version number.");
|
||
error = EINVAL;
|
||
goto out;
|
||
}
|
||
if (i > 0 && entry->priority >= entries[i - 1].priority) {
|
||
memorystatus_log_error("memorystatus_cmd_grp_set_freeze_priority: Entry list is not in descending order.");
|
||
error = EINVAL;
|
||
goto out;
|
||
}
|
||
}
|
||
#endif /* MACH_ASSERT */
|
||
|
||
lck_mtx_lock(&freezer_mutex);
|
||
|
||
tmp_entries = list->mfcl_list;
|
||
tmp_size = list->mfcl_length * sizeof(memorystatus_properties_freeze_entry_v1);
|
||
list->mfcl_list = entries;
|
||
list->mfcl_length = entry_count;
|
||
|
||
lck_mtx_unlock(&freezer_mutex);
|
||
|
||
entries = tmp_entries;
|
||
entries_size = tmp_size;
|
||
|
||
out:
|
||
kfree_data(entries, entries_size);
|
||
return error;
|
||
}
|
||
|
||
errno_t
|
||
memorystatus_cmd_grp_set_freeze_list(user_addr_t buffer, size_t buffer_size)
|
||
{
|
||
return set_freezer_candidate_list(buffer, buffer_size, &memorystatus_global_freeze_list);
|
||
}
|
||
|
||
errno_t
|
||
memorystatus_cmd_grp_set_demote_list(user_addr_t buffer, size_t buffer_size)
|
||
{
|
||
return set_freezer_candidate_list(buffer, buffer_size, &memorystatus_global_demote_list);
|
||
}
|
||
|
||
void
|
||
memorystatus_freezer_mark_ui_transition(proc_t p)
|
||
{
|
||
bool frozen = false, previous_focal_thaw = false, xpc_service = false, suspended = false;
|
||
proc_list_lock();
|
||
|
||
if (isSysProc(p)) {
|
||
goto out;
|
||
}
|
||
|
||
frozen = (p->p_memstat_state & P_MEMSTAT_FROZEN) != 0;
|
||
previous_focal_thaw = (p->p_memstat_state & P_MEMSTAT_FROZEN_FOCAL_THAW) != 0;
|
||
xpc_service = (p->p_memstat_state & P_MEMSTAT_FROZEN_XPC_SERVICE) != 0;
|
||
suspended = (p->p_memstat_state & P_MEMSTAT_SUSPENDED) != 0;
|
||
if (!suspended) {
|
||
if (frozen) {
|
||
if (!previous_focal_thaw) {
|
||
p->p_memstat_state |= P_MEMSTAT_FROZEN_FOCAL_THAW;
|
||
os_atomic_inc(&(memorystatus_freezer_stats.mfs_processes_thawed_fg), relaxed);
|
||
if (xpc_service) {
|
||
os_atomic_inc(&(memorystatus_freezer_stats.mfs_processes_thawed_fg_xpc_service), relaxed);
|
||
}
|
||
}
|
||
}
|
||
os_atomic_inc(&(memorystatus_freezer_stats.mfs_processes_fg_resumed), relaxed);
|
||
}
|
||
|
||
out:
|
||
proc_list_unlock();
|
||
}
|
||
|
||
#endif /* CONFIG_FREEZE */
|