gems-kernel/source/THIRDPARTY/xnu/bsd/kern/sys_recount.c

// Copyright (c) 2021 Apple Inc.  All rights reserved.
//
// @APPLE_OSREFERENCE_LICENSE_HEADER_START@
//
// This file contains Original Code and/or Modifications of Original Code
// as defined in and that are subject to the Apple Public Source License
// Version 2.0 (the 'License'). You may not use this file except in
// compliance with the License. The rights granted to you under the License
// may not be used to create, or enable the creation or redistribution of,
// unlawful or unlicensed copies of an Apple operating system, or to
// circumvent, violate, or enable the circumvention or violation of, any
// terms of an Apple operating system software license agreement.
//
// Please obtain a copy of the License at
// http://www.opensource.apple.com/apsl/ and read it before using this file.
//
// The Original Code and all software distributed under the License are
// distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
// EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
// INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
// Please see the License for the specific language governing rights and
// limitations under the License.
//
// @APPLE_OSREFERENCE_LICENSE_HEADER_END@

#include <kern/recount.h>
#include <machine/machine_routines.h>
#include <machine/smp.h>
#include <sys/proc_info.h>
#include <sys/resource_private.h>
#include <sys/sysproto.h>
#include <sys/systm.h>
#include <sys/types.h>

// Recount's BSD-specific implementation for syscalls.

#if CONFIG_PERVASIVE_CPI

static struct thsc_cpi
_usage_to_cpi(struct recount_usage *usage)
{
	return (struct thsc_cpi){
		       .tcpi_instructions = recount_usage_instructions(usage),
		       .tcpi_cycles = recount_usage_cycles(usage),
	};
}

static struct thsc_time_cpi
_usage_to_time_cpi(struct recount_usage *usage)
{
	return (struct thsc_time_cpi){
		       .ttci_instructions = recount_usage_instructions(usage),
		       .ttci_cycles = recount_usage_cycles(usage),
		       .ttci_system_time_mach = recount_usage_system_time_mach(usage),
		       .ttci_user_time_mach = usage->ru_metrics[RCT_LVL_USER].rm_time_mach,
	};
}

static struct thsc_time_energy_cpi
_usage_to_time_energy_cpi(struct recount_usage *usage)
{
	return (struct thsc_time_energy_cpi){
		       .ttec_instructions = recount_usage_instructions(usage),
		       .ttec_cycles = recount_usage_cycles(usage),
		       .ttec_system_time_mach = recount_usage_system_time_mach(usage),
		       .ttec_user_time_mach = usage->ru_metrics[RCT_LVL_USER].rm_time_mach,
#if CONFIG_PERVASIVE_ENERGY
		       .ttec_energy_nj = usage->ru_energy_nj,
#endif // CONFIG_PERVASIVE_ENERGY
	};
}

static recount_cpu_kind_t
_perflevel_index_to_cpu_kind(unsigned int perflevel)
{
#if __AMP__
	extern cluster_type_t cpu_type_for_perflevel(int perflevel);
	cluster_type_t cluster = cpu_type_for_perflevel(perflevel);
#else // __AMP__
	cluster_type_t cluster = CLUSTER_TYPE_SMP;
#endif // !__AMP__

	switch (cluster) {
	case CLUSTER_TYPE_SMP:
		// Default to first index for SMP.
		return (recount_cpu_kind_t)0;
#if __AMP__
	case CLUSTER_TYPE_E:
		return RCT_CPU_EFFICIENCY;
	case CLUSTER_TYPE_P:
		return RCT_CPU_PERFORMANCE;
#endif // __AMP__
	default:
		panic("recount: unexpected CPU type %d for perflevel %d", cluster,
		    perflevel);
	}
}

static int
_selfcounts(thread_selfcounts_kind_t kind, user_addr_t buf, size_t size)
{
	struct recount_usage usage = { 0 };
	boolean_t interrupt_state = ml_set_interrupts_enabled(FALSE);
	recount_current_thread_usage(&usage);
	ml_set_interrupts_enabled(interrupt_state);

	switch (kind) {
	case THSC_CPI: {
		struct thsc_cpi counts = _usage_to_cpi(&usage);
		return copyout(&counts, buf, MIN(sizeof(counts), size));
	}
	case THSC_TIME_CPI: {
		struct thsc_time_cpi counts = _usage_to_time_cpi(&usage);
		return copyout(&counts, buf, MIN(sizeof(counts), size));
	}
	case THSC_TIME_ENERGY_CPI: {
		struct thsc_time_energy_cpi counts = _usage_to_time_energy_cpi(&usage);
		return copyout(&counts, buf, MIN(sizeof(counts), size));
	}
	default:
		panic("recount: unexpected thread_selfcounts kind: %d", kind);
	}
}

static int
_selfcounts_perf_level(thread_selfcounts_kind_t kind, user_addr_t buf,
    size_t size)
{
	struct recount_usage usages[RCT_CPU_KIND_COUNT] = { 0 };
	boolean_t interrupt_state = ml_set_interrupts_enabled(FALSE);
	recount_current_thread_perf_level_usage(usages);
	ml_set_interrupts_enabled(interrupt_state);

	unsigned int cpu_types = ml_get_cpu_types();
	unsigned int level_count = __builtin_popcount(cpu_types);
	const size_t counts_len = MIN(MIN(recount_topo_count(RCT_TOPO_CPU_KIND),
	    RCT_CPU_KIND_COUNT), level_count);

	switch (kind) {
	case THSC_CPI_PER_PERF_LEVEL: {
		struct thsc_cpi counts[RCT_CPU_KIND_COUNT] = { 0 };
		for (unsigned int i = 0; i < counts_len; i++) {
			const recount_cpu_kind_t cpu_kind = _perflevel_index_to_cpu_kind(i);
			counts[i] = _usage_to_cpi(&usages[cpu_kind]);
		}
		return copyout(&counts, buf, MIN(sizeof(counts[0]) * counts_len, size));
	}
	case THSC_TIME_CPI_PER_PERF_LEVEL: {
		struct thsc_time_cpi counts[RCT_CPU_KIND_COUNT] = { 0 };
		for (unsigned int i = 0; i < counts_len; i++) {
			const recount_cpu_kind_t cpu_kind = _perflevel_index_to_cpu_kind(i);
			counts[i] = _usage_to_time_cpi(&usages[cpu_kind]);
		}
		return copyout(&counts, buf, MIN(sizeof(counts[0]) * counts_len, size));
	}
	case THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL: {
		struct thsc_time_energy_cpi counts[RCT_CPU_KIND_COUNT] = { 0 };
		for (unsigned int i = 0; i < counts_len; i++) {
			const recount_cpu_kind_t cpu_kind = _perflevel_index_to_cpu_kind(i);
			counts[i] = _usage_to_time_energy_cpi(&usages[cpu_kind]);
		}
		return copyout(&counts, buf, MIN(sizeof(counts[0]) * counts_len, size));
	}
	default:
		panic("recount: unexpected thread_selfcounts kind: %d", kind);
	}
}

int
thread_selfcounts(__unused struct proc *p,
    struct thread_selfcounts_args *uap, __unused int *ret_out)
{
	switch (uap->kind) {
	case THSC_CPI:
	case THSC_TIME_CPI:
	case THSC_TIME_ENERGY_CPI:
		return _selfcounts(uap->kind, uap->buf, uap->size);

	case THSC_CPI_PER_PERF_LEVEL:
	case THSC_TIME_CPI_PER_PERF_LEVEL:
	case THSC_TIME_ENERGY_CPI_PER_PERF_LEVEL:
		return _selfcounts_perf_level(uap->kind, uap->buf, uap->size);

	default:
		return ENOTSUP;
	}
}

static struct proc_threadcounts_data
_usage_to_proc_threadcounts(struct recount_usage *usage)
{
	return (struct proc_threadcounts_data){
		       .ptcd_instructions =  recount_usage_instructions(usage),
		       .ptcd_cycles =  recount_usage_cycles(usage),
		       .ptcd_system_time_mach =  recount_usage_system_time_mach(usage),
		       .ptcd_user_time_mach =  usage->ru_metrics[RCT_LVL_USER].rm_time_mach,
#if CONFIG_PERVASIVE_ENERGY
		       .ptcd_energy_nj = usage->ru_energy_nj,
#endif // CONFIG_PERVASIVE_ENERGY
	};
}

int
proc_pidthreadcounts(
	struct proc *p,
	uint64_t tid,
	user_addr_t uaddr,
	size_t usize,
	int *size_out)
{
	struct recount_usage usages[RCT_CPU_KIND_COUNT] = { 0 };
	// Keep this in sync with proc_threadcounts_data -- this one just has the
	// array length hard-coded to the maximum.
	struct {
		uint16_t counts_len;
		uint16_t reserved0;
		uint32_t reserved1;
		struct proc_threadcounts_data counts[RCT_CPU_KIND_COUNT];
	} counts = { 0 };

	task_t task = proc_task(p);
	if (task == TASK_NULL) {
		return ESRCH;
	}

	bool found = recount_task_thread_perf_level_usage(task, tid, usages);
	if (!found) {
		return ESRCH;
	}

	const size_t counts_len = MIN(recount_topo_count(RCT_TOPO_CPU_KIND),
	    RCT_CPU_KIND_COUNT);
	counts.counts_len = (uint16_t)counts_len;
	// The number of perflevels for this boot can be constrained by the `cpus=`
	// boot-arg, so determine the runtime number to prevent unexpected calls
	// into the machine-dependent layers from asserting.
	unsigned int cpu_types = ml_get_cpu_types();
	unsigned int level_count = __builtin_popcount(cpu_types);

	for (unsigned int i = 0; i < counts_len; i++) {
		if (i < level_count) {
			const recount_cpu_kind_t cpu_kind = _perflevel_index_to_cpu_kind(i);
			counts.counts[i] = _usage_to_proc_threadcounts(&usages[cpu_kind]);
		}
	}
	size_t copyout_size = MIN(sizeof(uint64_t) +
	    counts_len * sizeof(struct proc_threadcounts_data), usize);
	assert(copyout_size <= sizeof(counts));
	int error = copyout(&counts, uaddr, copyout_size);
	if (error == 0) {
		*size_out = (int)copyout_size;
	}
	return error;
}

#else // CONFIG_PERVASIVE_CPI

int
proc_pidthreadcounts(
	__unused struct proc *p,
	__unused uint64_t tid,
	__unused user_addr_t uaddr,
	__unused size_t usize,
	__unused int *ret_out)
{
	return ENOTSUP;
}

int
thread_selfcounts(__unused struct proc *p,
    __unused struct thread_selfcounts_args *uap, __unused int *ret_out)
{
	return ENOTSUP;
}

#endif // !CONFIG_PERVASIVE_CPI