android/kernel/sched/eas_plus.c

/*
 * Copyright (C) 2016 MediaTek Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See http://www.gnu.org/licenses/gpl-2.0.html for more details.
 */
#include <linux/stop_machine.h>
static inline unsigned long task_util(struct task_struct *p);
static int select_max_spare_capacity(struct task_struct *p, int target);
int cpu_eff_tp = 1024;
unsigned long long big_cpu_eff_tp = 1024;

#ifndef cpu_isolated
#define cpu_isolated(cpu) 0
#endif

#if defined(CONFIG_MACH_MT6763) || defined(CONFIG_MACH_MT6758)
/* cpu7 is L+ */
int l_plus_cpu = 7;
#else
int l_plus_cpu = -1;
#endif


#ifdef CONFIG_MTK_SCHED_EAS_POWER_SUPPORT

#if defined(CONFIG_MACH_MT6763) || defined(CONFIG_MACH_MT6758)
/* MT6763: 2 gears. cluster 0 & 1 is buck shared. */
static int share_buck[3] = {1, 0, 2};
#elif defined(CONFIG_MACH_MT6799)
/* MT6799: 3 gears. cluster 0 & 2 is buck shared. */
static int share_buck[3] = {2, 1, 0};
#elif defined(CONFIG_MACH_MT6765) || defined(CONFIG_MACH_MT6762)
static int share_buck[3] = {1, 0, 2};
#elif defined(CONFIG_MACH_MT6779)
static int share_buck[2] = {2, 1};
#define ARM_V8_2
int l_plus_cpu = -1;
#elif defined(CONFIG_MACH_MT6893) || \
	(defined(CONFIG_MACH_MT6885) && defined(CONFIG_MTK_SCHED_MULTI_GEARS))
static int share_buck[3] = {0, 2, 1};
#else
/* no buck shared */
static int share_buck[3] = {0, 1, 2};
#endif

#endif

#define CCI_ID (arch_get_nr_clusters())


static void
update_system_overutilized(struct lb_env *env)
{
	unsigned long group_util;
	bool intra_overutil = false;
	unsigned long min_capacity;
	struct sched_group *group = env->sd->groups;
	int this_cpu;
	int min_cap_orig_cpu;
	bool overutilized =  sd_overutilized(env->sd);
	int i;

	if (!sched_feat(SCHED_MTK_EAS))
		return;

	this_cpu = smp_processor_id();
	min_cap_orig_cpu = cpu_rq(this_cpu)->rd->min_cap_orig_cpu;
	if (min_cap_orig_cpu > -1)
		min_capacity = capacity_orig_of(min_cap_orig_cpu);
	else
		return;

	do {

		group_util = 0;

		for_each_cpu_and(i, sched_group_span(group), env->cpus) {

			if (cpu_isolated(i))
				continue;

			group_util += cpu_util(i);
		}

		/*
		 * A capacity base hint for over-utilization.
		 * Not to trigger system overutiled if heavy tasks
		 * in Big.cluster, so
		 * add the free room(20%) of Big.cluster is impacted which means
		 * system-wide over-utilization,
		 * that considers whole cluster not single cpu
		 */
		if (group->group_weight > 1 && (group->sgc->capacity * 1024 <
					group_util * capacity_margin)) {
			intra_overutil = true;
			break;
		}

		group = group->next;

	} while (group != env->sd->groups && !intra_overutil);

	if (overutilized != intra_overutil) {
		if (intra_overutil == true)
			set_sd_overutilized(env->sd);
		else
			clear_sd_overutilized(env->sd);
	}
}

bool is_intra_domain(int prev, int target)
{
#ifdef CONFIG_ARM64
	return (cpu_topology[prev].cluster_id ==
			cpu_topology[target].cluster_id);
#else
	return (cpu_topology[prev].socket_id ==
			cpu_topology[target].socket_id);
#endif
}

static int
___select_idle_sibling(struct task_struct *p, int prev_cpu, int new_cpu)
{
	if (sched_feat(SCHED_MTK_EAS)) {
#ifdef CONFIG_SCHED_TUNE
		bool prefer_idle = schedtune_prefer_idle(p) > 0;
#else
		bool prefer_idle = true;
#endif
		int idle_cpu;

		idle_cpu = find_best_idle_cpu(p, prefer_idle);
		if (idle_cpu >= 0)
			new_cpu = idle_cpu;
		else
			new_cpu = select_max_spare_capacity(p, new_cpu);
	} else
		new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);

	return new_cpu;
}

/* To find a CPU with max spare capacity in the same cluster with target */
static
int select_max_spare_capacity(struct task_struct *p, int target)
{
	unsigned long int max_spare_capacity = 0;
	int max_spare_cpu = -1;
	struct cpumask cls_cpus;
	int cid = arch_get_cluster_id(target); /* cid of target CPU */
	int cpu = task_cpu(p);
	struct cpumask *tsk_cpus_allow = &p->cpus_allowed;

	/* If the prevous cpu is cache affine and idle, choose it first. */
	if (cpu != l_plus_cpu && cpu != target &&
		cpus_share_cache(cpu, target) &&
		idle_cpu(cpu) && !cpu_isolated(cpu))
		return cpu;

	arch_get_cluster_cpus(&cls_cpus, cid);

	/* Otherwise, find a CPU with max spare-capacity in cluster */
	for_each_cpu_and(cpu, tsk_cpus_allow, &cls_cpus) {
		unsigned long int new_usage;
		unsigned long int spare_cap;

		if (!cpu_online(cpu))
			continue;

		if (cpu_isolated(cpu))
			continue;

#ifdef CONFIG_MTK_SCHED_INTEROP
		if (cpu_rq(cpu)->rt.rt_nr_running &&
			likely(!is_rt_throttle(cpu)))
			continue;
#endif

#ifdef CONFIG_SCHED_WALT
		if (walt_cpu_high_irqload(cpu))
			continue;
#endif

		if (idle_cpu(cpu))
			return cpu;

		new_usage = cpu_util(cpu) + task_util(p);

		if (new_usage >= capacity_of(cpu))
			spare_cap = 0;
		else    /* consider RT/IRQ capacity reduction */
			spare_cap = (capacity_of(cpu) - new_usage);

		/* update CPU with max spare capacity */
		if ((long int)spare_cap > (long int)max_spare_capacity) {
			max_spare_cpu = cpu;
			max_spare_capacity = spare_cap;
		}
	}

	/* if max_spare_cpu exist, choose it. */
	if (max_spare_cpu > -1)
		return max_spare_cpu;
	else
		return task_cpu(p);
}

/*
 * @p: the task want to be located at.
 *
 * Return:
 *
 * cpu id or
 * -1 if target CPU is not found
 */
int find_best_idle_cpu(struct task_struct *p, bool prefer_idle)
{
	int i;
	int best_idle_cpu = -1;
	struct cpumask *tsk_cpus_allow = &p->cpus_allowed;
	struct hmp_domain *domain;
	int domain_order = 0;
	int prefer_big = prefer_idle && (task_util(p) > stune_task_threshold);

	for_each_hmp_domain_L_first(domain) {
		for_each_cpu(i, &domain->possible_cpus) {

			/* tsk with prefer idle to find bigger idle cpu */
			if (!cpu_online(i) || cpu_isolated(i) ||
					!cpumask_test_cpu(i, tsk_cpus_allow))
				continue;

#ifdef CONFIG_MTK_SCHED_INTEROP
			if (cpu_rq(i)->rt.rt_nr_running &&
					likely(!is_rt_throttle(i)))
				continue;
#endif

			/* favoring tasks that prefer idle cpus
			 * to improve latency.
			 */
			if (idle_cpu(i)) {
				best_idle_cpu = i;
				if (!prefer_big) {
					goto find_idle_cpu;
				} else {
#ifdef CONFIG_MTK_SCHED_BL_FIRST
					if (domain_order == 1)
						goto find_idle_cpu;
#endif
				}
			}
		}

		domain_order++;
	}

find_idle_cpu:

	return best_idle_cpu;
}

static int init_cpu_info(void)
{
	int i;

	for (i = 0; i < nr_cpu_ids; i++) {
		unsigned long capacity = SCHED_CAPACITY_SCALE;

		if (cpu_core_energy(i)) {
			int idx = cpu_core_energy(i)->nr_cap_states - 1;

			capacity = cpu_core_energy(i)->cap_states[idx].cap;
		}
	}

	return 0;
}
late_initcall_sync(init_cpu_info)

#ifdef CONFIG_MTK_UNIFY_POWER
void set_sched_turn_point_cap(void)
{
	int turn_point_idx;
	struct hmp_domain *domain;
	int cpu;
	const struct sched_group_energy *sge_core;

	domain = list_entry(hmp_domains.prev, struct hmp_domain, hmp_domains);
	cpu = cpumask_first(&domain->possible_cpus);
	sge_core = cpu_core_energy(cpu);

	turn_point_idx = max(upower_get_turn_point() - 1, 0);
	cpu_eff_tp = sge_core->cap_states[turn_point_idx].cap;
}
#else
void set_sched_turn_point_cap(void)
{
	return;
}
#endif

#if defined(CONFIG_SCHED_HMP) || defined(CONFIG_MTK_IDLE_BALANCE_ENHANCEMENT)

/*
 * Heterogenous Multi-Processor (HMP) Global Load Balance
 */
static DEFINE_SPINLOCK(hmp_force_migration);

/*
 * For debugging purpose,
 * to depart functions of cpu_stop to make call_stack clear.
 */
static int hmp_idle_pull_cpu_stop(void *data)
{
	int ret;
	struct task_struct *p = ((struct rq *)data)->migrate_task;

	ret = active_load_balance_cpu_stop(data);
	put_task_struct(p);
	return ret;
}

static int
migrate_running_task(int this_cpu, struct task_struct *p, struct rq *target)
{
	unsigned long flags;
	unsigned int force = 0;

	/* now we have a candidate */
	raw_spin_lock_irqsave(&target->lock, flags);
	if (!target->active_balance &&
			(task_rq(p) == target) && !cpu_park(cpu_of(target)) &&
			p->state != TASK_DEAD) {
		get_task_struct(p);
		target->push_cpu = this_cpu;
		target->migrate_task = p;
		trace_sched_hmp_migrate(p, target->push_cpu, MIGR_IDLE_RUNNING);
#ifdef CONFIG_SCHED_HMP
		hmp_next_up_delay(&p->se, target->push_cpu);
#endif
		target->active_balance = MIGR_IDLE_RUNNING; /* idle pull */
		force = 1;
	}
	raw_spin_unlock_irqrestore(&target->lock, flags);
	if (force) {
		if (!stop_one_cpu_nowait(cpu_of(target),
					hmp_idle_pull_cpu_stop,
					target, &target->active_balance_work)) {
			put_task_struct(p); /* out of rq->lock */
			raw_spin_lock_irqsave(&target->lock, flags);
			target->active_balance = 0;
			target->migrate_task = NULL;
			force = 0;
			raw_spin_unlock_irqrestore(&target->lock, flags);
		}
	}

	return force;
}
#endif

unsigned long cluster_max_capacity(void)
{
	struct hmp_domain *domain;
	unsigned int max_capacity = 0;

	for_each_hmp_domain_L_first(domain) {
		int cpu;
		unsigned long capacity;

		cpu = cpumask_first(&domain->possible_cpus);
		capacity = capacity_of(cpu);

		if (capacity > max_capacity)
			max_capacity = capacity;
	}

	return max_capacity;
}

inline unsigned long task_uclamped_min_w_ceiling(struct task_struct *p)
{
	unsigned long max_capacity = cluster_max_capacity();

	return min_t(unsigned int, uclamp_task_effective_util(p, UCLAMP_MIN),
			max_capacity);
}

/* Calculte util with DVFS margin */
inline unsigned int freq_util(unsigned long util)
{
	return util * capacity_margin / SCHED_CAPACITY_SCALE;
}

#ifdef CONFIG_MTK_IDLE_BALANCE_ENHANCEMENT
bool idle_lb_enhance(struct task_struct *p, int cpu)
{
	int target_capacity = capacity_orig_of(cpu);

	if (schedtune_prefer_idle(p))
		return 1;

	if (uclamp_task_effective_util(p, UCLAMP_MIN) > target_capacity)
		return 1;

	return 0;
}

/* must hold runqueue lock for queue se is currently on */
static const int idle_prefer_max_tasks = 5;
static struct sched_entity
*get_idle_prefer_task(int cpu, int target_cpu, int check_min_cap,
		struct task_struct **backup_task, int *backup_cpu)
{
	int num_tasks = idle_prefer_max_tasks;
	const struct cpumask *hmp_target_mask = NULL;
	int src_capacity;
	unsigned int util_min;
	struct cfs_rq *cfs_rq;
	struct sched_entity *se;

	if (target_cpu >= 0)
		hmp_target_mask = cpumask_of(target_cpu);
	else
		return NULL;

	/* The currently running task is not on the runqueue
	 *	a. idle prefer
	 *	b. task_capacity > belonged CPU
	 */
	src_capacity = capacity_orig_of(cpu);
	cfs_rq = &cpu_rq(cpu)->cfs;
	se = __pick_first_entity(cfs_rq);
#ifdef CONFIG_FAIR_GROUP_SCHED
	if (!se && cfs_rq->h_nr_running > 1) {
		se = cfs_rq->curr;
		if (se && group_cfs_rq(se)) {
			se = __pick_first_entity(group_cfs_rq(se));
		}
	}
#endif	/* CONFIG_FAIR_GROUP_SCHED */
	while (num_tasks && se) {
		if (entity_is_task(se) &&
		    cpumask_intersects(hmp_target_mask,
				       &(task_of(se)->cpus_allowed))) {
			struct task_struct *p;

			p = task_of(se);
			util_min = uclamp_task_effective_util(p, UCLAMP_MIN);

#ifdef CONFIG_MTK_SCHED_BOOST
			if (!task_prefer_match_on_cpu(p, cpu, target_cpu))
				return se;
#endif

			if (check_min_cap && util_min >= src_capacity)
				return se;

			if (schedtune_prefer_idle(task_of(se)) &&
					cpu_rq(cpu)->nr_running > 1) {
				if (!check_min_cap)
					return se;

				if (backup_task && !*backup_task) {
					*backup_cpu = cpu;
					/* get task and selection inside
					 * rq lock
					 */
					*backup_task = task_of(se);
					get_task_struct(*backup_task);
				}
			}
		}
		se = __pick_next_entity(se);
		num_tasks--;
	}

	return NULL;
}

static void
hmp_slowest_idle_prefer_pull(int this_cpu, struct task_struct **p,
			     struct rq **target)
{
	int cpu, backup_cpu;
	struct sched_entity *se = NULL;
	struct task_struct  *backup_task = NULL;
	struct hmp_domain *domain;
	struct list_head *pos;
	int selected = 0;
	struct rq *rq;
	unsigned long flags;
	int check_min_cap;

	/* 1. select a runnable task
	 *     idle prefer
	 *
	 *     order: fast to slow hmp domain
	 */
	check_min_cap = 0;
	list_for_each(pos, &hmp_domains) {
		domain = list_entry(pos, struct hmp_domain, hmp_domains);

		for_each_cpu(cpu, &domain->cpus) {
			if (cpu == this_cpu)
				continue;

			rq = cpu_rq(cpu);
			raw_spin_lock_irqsave(&rq->lock, flags);

			se = get_idle_prefer_task(cpu, this_cpu,
				check_min_cap, &backup_task, &backup_cpu);
			if (se && entity_is_task(se) &&
			    cpumask_test_cpu(this_cpu,
					     &(task_of(se))->cpus_allowed)) {
				selected = 1;
				/* get task and selection inside rq lock  */
				*p = task_of(se);
				get_task_struct(*p);

				*target = rq;
			}

			raw_spin_unlock_irqrestore(&rq->lock, flags);

			if (selected) {
				/* To put task out of rq lock */
				if (backup_task)
					put_task_struct(backup_task);
				return;
			}
		}
	}

	if (backup_task) {
		*target = cpu_rq(backup_cpu);
		return;
	}
}

DECLARE_PER_CPU(struct hmp_domain *, hmp_cpu_domain);
static void
hmp_fastest_idle_prefer_pull(int this_cpu, struct task_struct **p,
						struct rq **target)
{
	int cpu, backup_cpu;
	struct sched_entity *se = NULL;
	struct task_struct  *backup_task = NULL;
	struct hmp_domain *hmp_domain = NULL, *domain;
	struct list_head *pos;
	int selected = 0;
	struct rq *rq;
	unsigned long flags;
	int target_capacity;
	int check_min_cap;
	int turning;

	hmp_domain = per_cpu(hmp_cpu_domain, this_cpu);

	/* 1. select a runnable task
	 *
	 * first candidate:
	 *     capacity_min in slow domain
	 *
	 *     order: target->next to slow hmp domain
	 */
	check_min_cap = 1;
	list_for_each(pos, &hmp_domain->hmp_domains) {
		domain = list_entry(pos, struct hmp_domain, hmp_domains);

		for_each_cpu(cpu, &domain->cpus) {
			if (cpu == this_cpu)
				continue;

			rq = cpu_rq(cpu);
			raw_spin_lock_irqsave(&rq->lock, flags);

			se = get_idle_prefer_task(cpu, this_cpu,
				check_min_cap, &backup_task, &backup_cpu);
			if (se && entity_is_task(se) &&
			    cpumask_test_cpu(this_cpu,
					     &(task_of(se))->cpus_allowed)) {
				selected = 1;
				/* get task and selection inside rq lock  */
				*p = task_of(se);
				get_task_struct(*p);

				*target = rq;
			}

			raw_spin_unlock_irqrestore(&rq->lock, flags);

			if (selected) {
				/* To put task out of rq lock */
				if (backup_task)
					put_task_struct(backup_task);
				return;
			}
		}

		if (list_is_last(pos, &hmp_domains))
			break;
	}

	/* backup candidate:
	 *     idle prefer
	 *
	 *     order: fastest to target hmp domain
	 */
	check_min_cap = 0;
	list_for_each(pos, &hmp_domains) {
		domain = list_entry(pos, struct hmp_domain, hmp_domains);

		for_each_cpu(cpu, &domain->cpus) {
			if (cpu == this_cpu)
				continue;

			rq = cpu_rq(cpu);
			raw_spin_lock_irqsave(&rq->lock, flags);

			se = get_idle_prefer_task(cpu, this_cpu,
				check_min_cap, &backup_task, &backup_cpu);

			if (se && entity_is_task(se) &&
			    cpumask_test_cpu(this_cpu,
					     &(task_of(se)->cpus_allowed))) {
				selected = 1;
				/* get task and selection inside rq lock  */
				*p = task_of(se);
				get_task_struct(*p);

				*target = rq;
			}

			raw_spin_unlock_irqrestore(&rq->lock, flags);

			if (selected) {
				/* To put task out of rq lock */
				if (backup_task)
					put_task_struct(backup_task);
				return;
			}
		}

		if (cpumask_test_cpu(this_cpu, &domain->cpus))
			break;
	}

	if (backup_task) {
		*p = backup_task;
		*target = cpu_rq(backup_cpu);
		return;
	}

	/* 2. select a running task
	 *     order: target->next to slow hmp domain
	 * 3. turning = true, pick a runnable task from slower domain
	 */
	turning = check_freq_turning();
	list_for_each(pos, &hmp_domain->hmp_domains) {
		domain = list_entry(pos, struct hmp_domain, hmp_domains);

		for_each_cpu(cpu, &domain->cpus) {
			if (cpu == this_cpu)
				continue;

			rq = cpu_rq(cpu);
			raw_spin_lock_irqsave(&rq->lock, flags);

			se = rq->cfs.curr;
			if (!se) {
				raw_spin_unlock_irqrestore(&rq->lock, flags);
				continue;
			}
			if (!entity_is_task(se)) {
				struct cfs_rq *cfs_rq;

				cfs_rq = group_cfs_rq(se);
				while (cfs_rq) {
					se = cfs_rq->curr;
					if (!entity_is_task(se))
						cfs_rq = group_cfs_rq(se);
					else
						cfs_rq = NULL;
				}
			}

			target_capacity = capacity_orig_of(cpu);
			if (se && entity_is_task(se) &&
			     (uclamp_task_effective_util(task_of(se),
				UCLAMP_MIN) >= target_capacity) &&
			     cpumask_test_cpu(this_cpu,
					      &((task_of(se))->cpus_allowed))) {
				selected = 1;
				/* get task and selection inside rq lock  */
				*p = task_of(se);
				get_task_struct(*p);

				*target = rq;
			}

			raw_spin_unlock_irqrestore(&rq->lock, flags);

			if (selected) {
				/* To put task out of rq lock */
				if (backup_task)
					put_task_struct(backup_task);
				return;
			}

			if (turning && !backup_task) {
				const struct cpumask *hmp_target_mask = NULL;
				struct cfs_rq *cfs_rq;
				struct sched_entity *se;

				raw_spin_lock_irqsave(&rq->lock, flags);

				hmp_target_mask = cpumask_of(this_cpu);
				cfs_rq = &rq->cfs;
				se = __pick_first_entity(cfs_rq);
				if (se && entity_is_task(se) &&
					    cpumask_intersects(hmp_target_mask,
						&(task_of(se)->cpus_allowed))) {
					backup_cpu = cpu;
					/* get task and selection inside
					 * rq lock
					 */
					backup_task = task_of(se);
					get_task_struct(backup_task);
				}
				raw_spin_unlock_irqrestore(&rq->lock, flags);
			}
		}

		if (list_is_last(pos, &hmp_domains))
			break;
	}

	if (backup_task) {
		*p = backup_task;
		*target = cpu_rq(backup_cpu);
		return;
	}
}

/*
 * rq: src rq
 */
static int
migrate_runnable_task(struct task_struct *p, int dst_cpu,
					struct rq *rq)
{
	struct rq_flags rf;
	int moved = 0;
	int src_cpu = cpu_of(rq);

	if (!raw_spin_trylock(&p->pi_lock))
		return moved;

	rq_lock(rq, &rf);

	/* Are both target and busiest cpu online */
	if (!cpu_online(src_cpu) || !cpu_online(dst_cpu) ||
		cpu_isolated(src_cpu) || cpu_isolated(dst_cpu))
		goto out_unlock;

	/* Task has migrated meanwhile, abort forced migration */
	/* can't migrate running task */
	if (task_running(rq, p))
		goto out_unlock;

	/*
	 * If task_rq(p) != rq, it cannot be migrated here, because we're
	 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
	 * we're holding p->pi_lock.
	 */
	if (task_rq(p) == rq) {
		if (task_on_rq_queued(p)) {
			rq = __migrate_task(rq, &rf, p, dst_cpu);
			moved = 1;
		}
	}

out_unlock:
	rq_unlock(rq, &rf);
	raw_spin_unlock(&p->pi_lock);

	return moved;
}

static unsigned int aggressive_idle_pull(int this_cpu)
{
	int moved = 0;
	struct rq *target = NULL;
	struct task_struct *p = NULL;

	if (!sched_smp_initialized)
		return 0;

	if (!spin_trylock(&hmp_force_migration))
		return 0;

	/*
	 * aggressive idle balance for min_cap/idle_prefer
	 */
	if (hmp_cpu_is_slowest(this_cpu)) {
		hmp_slowest_idle_prefer_pull(this_cpu, &p, &target);
		if (p) {
			trace_sched_hmp_migrate(p, this_cpu, 0x10);
			moved = migrate_runnable_task(p, this_cpu, target);
			if (moved)
				goto done;
		}
	} else {
		hmp_fastest_idle_prefer_pull(this_cpu, &p, &target);
		if (p) {
			trace_sched_hmp_migrate(p, this_cpu, 0x10);
			moved = migrate_runnable_task(p, this_cpu, target);
			if (moved)
				goto done;

			moved = migrate_running_task(this_cpu, p, target);
		}
	}

done:
	spin_unlock(&hmp_force_migration);
	if (p)
		put_task_struct(p);

	return moved;
}

#else
bool idle_lb_enhance(struct task_struct *p, int cpu)
{
	return 0;
}

static unsigned int aggressive_idle_pull(int this_cpu)
{
	return 0;
}
#endif

#ifdef CONFIG_UCLAMP_TASK
static __always_inline
unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,
					struct task_struct *p)
{
	unsigned long min_util = rq->uclamp.value[UCLAMP_MIN];
	unsigned long max_util = rq->uclamp.value[UCLAMP_MAX];

	if (p) {
		min_util = max_t(unsigned long, min_util,
		  (unsigned long)uclamp_task_effective_util(p, UCLAMP_MIN));
		max_util = max_t(unsigned long, max_util,
		  (unsigned long)uclamp_task_effective_util(p, UCLAMP_MAX));
	}

	/*
	 * Since CPU's {min,max}_util clamps are MAX aggregated considering
	 * RUNNABLE tasks with_different_ clamps, we can end up with an
	 * inversion. Fix it now when the clamps are applied.
	 */
	if (unlikely(min_util >= max_util))
		return min_util;

	return clamp(util, min_util, max_util);
}
#endif

#ifdef CONFIG_MTK_SCHED_EAS_POWER_SUPPORT
#define fits_capacity(cap, max) ((cap) * capacity_margin < (max) * 1024)


static unsigned long __cpu_norm_sumutil(unsigned long util,
					unsigned long capacity)
{
	return  (util << SCHED_CAPACITY_SHIFT)/capacity;
}

struct sg_state {
	int cid;
	int cap_idx;
	unsigned long cap;
	unsigned long volt;
	unsigned long max_util;
	unsigned long sum_util;
};

/*
 * compute_energy(): Estimates the energy that @pd would consume if @p was
 * migrated to @dst_cpu. compute_energy() predicts what will be the utilization
 * landscape of @pd's CPUs after the task migration, and uses the Energy Model
 * to compute what would be the energy if we decided to actually migrate that
 * task.
 */
static int
update_sg_util(struct task_struct *p, int dst_cpu,
		const struct cpumask *sg_mask, struct sg_state *sg_env)
{
	int cpu = cpumask_first(sg_mask);
	struct sched_domain *sd;
	const struct sched_group *sg;
	const struct sched_group_energy *sge;
	unsigned long new_util;
	int idx, max_idx;

	sg_env->sum_util = 0;
	sg_env->max_util = 0;

	sge = cpu_core_energy(cpu); /* for CPU */
	/*
	 * The capacity state of CPUs of the current rd can be driven by CPUs
	 * of another rd if they belong to the same pd. So, account for the
	 * utilization of these CPUs too by masking pd with cpu_online_mask
	 * instead of the rd span.
	 *
	 * If an entire pd is outside of the current rd, it will not appear in
	 * its pd list and will not be accounted by compute_energy().
	 */
	for_each_cpu_and(cpu, sg_mask, cpu_online_mask) {
		unsigned long cpu_util, cpu_boosted_util;
		struct task_struct *tsk = cpu == dst_cpu ? p : NULL;

		cpu_util = cpu_util_without(cpu, p);
		cpu_boosted_util = uclamp_rq_util_with(cpu_rq(cpu), cpu_util, p);

		if (tsk)
			cpu_util += task_util_est(p);

		sg_env->sum_util += cpu_util;
		sg_env->max_util = max(sg_env->max_util, cpu_boosted_util);
	}

	/* default is max_cap if we don't find a match */
	max_idx = sge->nr_cap_states - 1;
	sg_env->cap_idx = max_idx;
	sg_env->cap = sge->cap_states[max_idx].cap;

	new_util = sg_env->max_util * capacity_margin >>  SCHED_CAPACITY_SHIFT;
	new_util = min_t(unsigned long, new_util,
		(unsigned long) sge->cap_states[sge->nr_cap_states-1].cap);

	for (idx = 0; idx < sge->nr_cap_states; idx++) {
		if (sge->cap_states[idx].cap >= new_util) {
			/* Keep track of SG's capacity */
			sg_env->cap_idx	= idx;
			sg_env->cap = sge->cap_states[idx].cap;
			sg_env->volt = sge->cap_states[idx].volt;
			break;
		}
	}

	mt_sched_printf(sched_eas_energy_calc,
		"dst_cpu=%d mask=0x%lx sum_util=%lu max_util=%lu new_util=%lu (idx=%d cap=%ld volt=%ld)",
		dst_cpu, sg_mask->bits[0], sg_env->sum_util, sg_env->max_util,
		new_util, sg_env->cap_idx, sg_env->cap, sg_env->volt);

	return 1;
}

unsigned int share_buck_lkg_idx(const struct sched_group_energy *_sge,
				int cpu_idx, unsigned long v_max)
{
	int co_buck_lkg_idx = _sge->nr_cap_states - 1;
	int idx;

	for (idx = cpu_idx; idx < _sge->nr_cap_states; idx++) {
		if (_sge->cap_states[idx].volt >= v_max) {
			co_buck_lkg_idx = idx;
			break;
		}
	}

	return co_buck_lkg_idx;
}

#define VOLT_SCALE 10
void calc_pwr(int sd_level, const struct sched_group_energy *_sge,
		int cap_idx, unsigned long volt, unsigned long co_volt,
		unsigned long *dyn_pwr, unsigned long *lkg_pwr)
{
	unsigned long int volt_factor = 1;

	if (co_volt > volt) {
		/*
		 * calculated power with share-buck impact
		 *
		 * dynamic power = F*V^2
		 *
		 * dyn_pwr  = current_power * (v_max/v_min)^2
		 * lkg_pwr = tlb[idx of v_max].leak;
		 */
		unsigned long v_max = co_volt;
		unsigned long v_min = volt;
		int lkg_idx = _sge->lkg_idx;
		int co_buck_lkg_idx;

		volt_factor = ((v_max*v_max) << VOLT_SCALE) /
				(v_min*v_min);
		*dyn_pwr = (_sge->cap_states[cap_idx].dyn_pwr *
				volt_factor) >> VOLT_SCALE;
		co_buck_lkg_idx = share_buck_lkg_idx(_sge, cap_idx, v_max);
		*lkg_pwr = _sge->cap_states[co_buck_lkg_idx].lkg_pwr[lkg_idx];

		trace_sched_busy_power(sd_level, cap_idx,
				_sge->cap_states[cap_idx].dyn_pwr, volt_factor,
				*dyn_pwr, co_buck_lkg_idx, *lkg_pwr,
				*dyn_pwr + *lkg_pwr);
	} else {
		/* No share buck impact */
		int lkg_idx = _sge->lkg_idx;

		*dyn_pwr = _sge->cap_states[cap_idx].dyn_pwr;
		*lkg_pwr = _sge->cap_states[cap_idx].lkg_pwr[lkg_idx];

		trace_sched_busy_power(sd_level, cap_idx, *dyn_pwr,
					volt_factor, *dyn_pwr, cap_idx,
					*lkg_pwr, *dyn_pwr + *lkg_pwr);

	}
}

/**
 * em_sg_energy() - Estimates the energy consumed by the CPUs of a perf. domain
 * @sd		: performance domain for which energy has to be estimated
 * @max_util	: highest utilization among CPUs of the domain
 * @sum_util	: sum of the utilization of all CPUs in the domain
 *
 * Return: the sum of the energy consumed by the CPUs of the domain assuming
 * a capacity state satisfying the max utilization of the domain.
 */
static inline unsigned long compute_energy_sg(const struct cpumask *sg_cpus,
			struct sg_state *sg_env, struct sg_state *share_env)
{
	int cpu;
	const struct sched_group_energy *_sge;
	unsigned long dyn_pwr, lkg_pwr;
	unsigned long dyn_egy, lkg_egy;
	unsigned long total_energy;
	unsigned long sg_util;

	cpu = cpumask_first(sg_cpus);
	_sge = cpu_core_energy(cpu); /* for CPU */
	calc_pwr(0, _sge,
		sg_env->cap_idx, sg_env->volt, share_env->volt,
		&dyn_pwr, &lkg_pwr);

	sg_util = __cpu_norm_sumutil(sg_env->sum_util, sg_env->cap);
	dyn_egy = sg_util * dyn_pwr;
	lkg_egy = SCHED_CAPACITY_SCALE * lkg_pwr;
	total_energy = dyn_egy + lkg_egy;

	mt_sched_printf(sched_eas_energy_calc,
			"sg_util=%lu dyn_egy=%d lkg_egy=%d (cost=%d) mask=0x%lx",
			sg_util,
			(int)dyn_egy, (int)lkg_egy, (int)total_energy,
			sg_cpus->bits[0]);

	return total_energy;
}

bool is_share_buck(int cid, int *co_buck_cid)
{
	bool ret = false;

	if (share_buck[cid] != cid) {
		*co_buck_cid = share_buck[cid];
		ret = true;
	}

	return ret;
}

static long
compute_energy_enhanced(struct task_struct *p, int dst_cpu,
				struct sched_group *sg)
{
	int cid, share_cid, cpu;
	struct sg_state sg_env, share_env;
	const struct cpumask *sg_cpus;
	struct cpumask share_cpus;
	unsigned long total_energy = 0;

	share_env.volt = 0;
	sg_cpus = sched_group_span(sg);
	cpu = cpumask_first(sg_cpus);
#ifdef CONFIG_ARM64
	cid = cpu_topology[cpu].cluster_id;
#else
	cid = cpu_topology[cpu].socket_id;
#endif
	if (!update_sg_util(p, dst_cpu, sg_cpus, &sg_env))
		return 0;

	if (is_share_buck(cid, &share_cid)) {
		arch_get_cluster_cpus(&share_cpus, share_cid);
		if (!update_sg_util(p, dst_cpu, &share_cpus, &share_env))
			return 0;

		total_energy += compute_energy_sg(&share_cpus, &share_env,
							&sg_env);
	}

	total_energy += compute_energy_sg(sg_cpus, &sg_env, &share_env);

	return total_energy;
}

static int find_energy_efficient_cpu_enhanced(struct task_struct *p,
					int this_cpu, int prev_cpu, int sync)
{
	unsigned long prev_energy = 0;
	unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
	int max_spare_cap_cpu_ls = prev_cpu;
	long max_spare_cap_ls = LONG_MIN;
	unsigned long target_cap, cpu_cap, util, wake_util;
	bool boosted, prefer_idle = false;
	unsigned int min_exit_lat = UINT_MAX;
	long sys_max_spare_cap = LONG_MIN;
	int sys_max_spare_cap_cpu = -1;
	int best_energy_cpu = prev_cpu;
	struct cpuidle_state *idle;
	struct sched_domain *sd;
	struct sched_group *sg;

	if (sysctl_sched_sync_hint_enable && sync) {
		if (cpumask_test_cpu(this_cpu, &p->cpus_allowed) &&
			!cpu_isolated(this_cpu)) {
			return this_cpu;
		}
	}

	sd = rcu_dereference(per_cpu(sd_ea, this_cpu));
	if (!sd)
		return -1;

	if (!boosted_task_util(p))
		return -1;

	prefer_idle = schedtune_prefer_idle(p);
	boosted = (schedtune_task_boost(p) > 0) || (uclamp_task_effective_util(p, UCLAMP_MIN) > 0);
	target_cap = boosted ? 0 : ULONG_MAX;

	sg = sd->groups;
	do {
		unsigned long cur_energy = 0, cur_delta = 0;
		unsigned long base_energy_sg;
		long spare_cap, max_spare_cap = LONG_MIN;
		int max_spare_cap_cpu = -1, best_idle_cpu = -1;
		int cpu;

		/* compute the ''base' energy of the sg, without @p*/
		base_energy_sg = compute_energy_enhanced(p, -1, sg);
		for_each_cpu_and(cpu, &p->cpus_allowed, sched_group_span(sg)) {

			if (cpu_isolated(cpu))
				continue;
#ifdef CONFIG_MTK_SCHED_INTEROP
			if (cpu_rq(cpu)->rt.rt_nr_running &&
				likely(!is_rt_throttle(cpu)))
				continue;
#endif

			/* Skip CPUs that will be overutilized. */
			wake_util = (prefer_idle && idle_cpu(cpu)) ? 0 : cpu_util_without(cpu, p);
			util = wake_util + task_util_est(p);
			cpu_cap = capacity_of(cpu);
			spare_cap = cpu_cap - util;
			if (spare_cap > sys_max_spare_cap) {
				sys_max_spare_cap = spare_cap;
				sys_max_spare_cap_cpu = cpu;
			}

			/*
			 * Skip CPUs that cannot satisfy the capacity request.
			 * IOW, placing the task there would make the CPU
			 * overutilized. Take uclamp into account to see how
			 * much capacity we can get out of the CPU; this is
			 * aligned with schedutil_cpu_util().
			 */
			util = uclamp_rq_util_with(cpu_rq(cpu), util, p);
			if (!fits_capacity(util, cpu_cap))
				continue;

			/* Always use prev_cpu as a candidate. */
			if (cpu == prev_cpu &&
			    (!prefer_idle || (prefer_idle && idle_cpu(cpu)))) {
				prev_energy = compute_energy_enhanced(p,
								prev_cpu, sg);
				prev_delta = prev_energy - base_energy_sg;
				best_delta = min(best_delta, prev_delta);
			}

			/*
			 * Find the CPU with the maximum spare capacity in
			 * the performance domain
			 */
			spare_cap = cpu_cap - util;
			if (spare_cap > max_spare_cap) {
				max_spare_cap = spare_cap;
				max_spare_cap_cpu = cpu;
			}

			if (!prefer_idle)
				continue;

			if (idle_cpu(cpu)) {
				cpu_cap = capacity_orig_of(cpu);
				if (!boosted && cpu_cap > target_cap)
					continue;
				idle = idle_get_state(cpu_rq(cpu));
				if (idle && idle->exit_latency >= min_exit_lat &&
						cpu_cap == target_cap)
					continue;

				if (idle)
					min_exit_lat = idle->exit_latency;
				target_cap = cpu_cap;
				best_idle_cpu = cpu;

			} else if (spare_cap > max_spare_cap_ls) {
				max_spare_cap_ls = spare_cap;
				max_spare_cap_cpu_ls = cpu;
			}
		}

		if (!prefer_idle && max_spare_cap_cpu >= 0 &&
					max_spare_cap_cpu != prev_cpu) {
			cur_energy = compute_energy_enhanced(p,
							max_spare_cap_cpu, sg);
			cur_delta = cur_energy - base_energy_sg;
			if (cur_delta < best_delta) {
				best_delta = cur_delta;
				best_energy_cpu = max_spare_cap_cpu;
			}
		}

		if (prefer_idle && best_idle_cpu >= 0 &&
					best_idle_cpu != prev_cpu) {
			cur_energy = compute_energy_enhanced(p,
							best_idle_cpu, sg);
			cur_delta = cur_energy - base_energy_sg;
			if (cur_delta < best_delta) {
				best_delta = cur_delta;
				best_energy_cpu = best_idle_cpu;
			}
		}

		mt_sched_printf(sched_eas_energy_calc,
		    "prev_cpu=%d base_energy=%lu prev_energy=%lu prev_delta=%d",
		    prev_cpu, base_energy_sg, prev_energy, (int)prev_delta);

		mt_sched_printf(sched_eas_energy_calc,
		    "max_spare_cap_cpu=%d best_idle_cpu=%d cur_energy=%lu cur_delta=%d",
			max_spare_cap_cpu, best_idle_cpu, cur_energy, (int)cur_delta);

	} while (sg = sg->next, sg != sd->groups);

	/*
	 * Pick the best CPU if prev_cpu cannot be used, or it it saves energy
	 * used by prev_cpu.
	 */
	if (prev_delta == ULONG_MAX) {
		/* All cpu failed on !fit_capacity, use sys_max_spare_cap_cpu */
		if (best_energy_cpu == prev_cpu)
			return sys_max_spare_cap_cpu;
		else
			return best_energy_cpu;
	}

	if ((prev_delta - best_delta) > 0)
		return best_energy_cpu;

	return prev_cpu;
}

static int __find_energy_efficient_cpu(struct sched_domain *sd,
				     struct task_struct *p,
				     int cpu, int prev_cpu,
				     int sync)
{
	int num_cluster = arch_get_nr_clusters();

	if (num_cluster <= 2)
		return find_energy_efficient_cpu(sd, p, cpu, prev_cpu, sync);
	else
		return find_energy_efficient_cpu_enhanced(p, cpu, prev_cpu, sync);
}

/*
 * group_norm_util() returns the approximated group util relative to it's
 * current capacity (busy ratio) in the range [0..SCHED_CAPACITY_SCALE] for use
 * in energy calculations. Since task executions may or may not overlap in time
 * in the group the true normalized util is between max(cpu_norm_util(i)) and
 * sum(cpu_norm_util(i)) when iterating over all cpus in the group, i. The
 * latter is used as the estimate as it leads to a more pessimistic energy
 * estimate (more busy).
 */
static unsigned
long group_norm_util(struct energy_env *eenv, int cpu_idx)
{
	struct sched_group *sg = eenv->sg;
	int cpu_id = group_first_cpu(sg);
#ifdef CONFIG_ARM64
	int cid = cpu_topology[cpu_id].cluster_id;
#else
	int cid = cpu_topology[cpu_id].socket_id;
#endif
	unsigned long capacity = eenv->cpu[cpu_idx].cap[cid];
	unsigned long util, util_sum = 0;
	int cpu;

	for_each_cpu(cpu, sched_group_span(eenv->sg)) {
		util = cpu_util_without(cpu, eenv->p);

		/*
		 * If we are looking at the target CPU specified by the eenv,
		 * then we should add the (estimated) utilization of the task
		 * assuming we will wake it up on that CPU.
		 */
		if (unlikely(cpu == eenv->cpu[cpu_idx].cpu_id))
			util += eenv->util_delta;

		util_sum += __cpu_norm_util(util, capacity);

		trace_group_norm_util(cpu_idx, cpu, cid, util_sum,
			__cpu_norm_util(util, capacity), eenv->util_delta,
			util, capacity);
	}

	if (util_sum > SCHED_CAPACITY_SCALE)
		return SCHED_CAPACITY_SCALE;
	return util_sum;
}
#endif


#ifdef CONFIG_MTK_SCHED_EAS_POWER_SUPPORT
static unsigned long
mtk_cluster_max_usage(int cid, struct energy_env *eenv, int cpu_idx,
			int *max_cpu)
{
	unsigned long util, max_util = 0;
	int cpu = -1;
	struct cpumask cls_cpus;

	*max_cpu = -1;

	arch_get_cluster_cpus(&cls_cpus, cid);

	for_each_cpu(cpu, &cls_cpus) {

		if (!cpu_online(cpu))
			continue;

		util = cpu_util_without(cpu, eenv->p);

		/*
		 * If we are looking at the target CPU specified by the eenv,
		 * then we should add the (estimated) utilization of the task
		 * assuming we will wake it up on that CPU.
		 */
		if (unlikely(cpu == eenv->cpu[cpu_idx].cpu_id))
			util += eenv->util_delta;

		if (util >= max_util) {
			max_util = util;
			*max_cpu = cpu;
		}
	}

	return max_util;
}

void mtk_cluster_capacity_idx(int cid, struct energy_env *eenv, int cpu_idx)
{
	int cpu;
	unsigned long util = mtk_cluster_max_usage(cid, eenv, cpu_idx, &cpu);
	unsigned long new_capacity = util;
	struct sched_domain *sd;
	struct sched_group *sg;
	const struct sched_group_energy *sge;
	int idx, max_idx;

	if (cpu == -1) { /* maybe no online CPU */
		printk_deferred("sched: %s no online CPU", __func__);
		return;
	}

	sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd);
	if (sd) {
		sg = sd->groups;
		sge = sg->sge;
	} else{
		printk_deferred("sched: %s no sd", __func__);
		return;
	}

	max_idx = sge->nr_cap_states - 1;

	/* default is max_cap if we don't find a match */
	eenv->cpu[cpu_idx].cap_idx[cid] = max_idx;
	eenv->cpu[cpu_idx].cap[cid] = sge->cap_states[max_idx].cap;

	/* OPP idx to refer capacity margin */
	new_capacity = util * capacity_margin >> SCHED_CAPACITY_SHIFT;
	new_capacity = min(new_capacity,
		(unsigned long) sge->cap_states[sge->nr_cap_states-1].cap);

	for (idx = 0; idx < sge->nr_cap_states; idx++) {
		if (sge->cap_states[idx].cap >= new_capacity) {
			/* Keep track of SG's capacity */
			eenv->cpu[cpu_idx].cap_idx[cid] = idx;
			eenv->cpu[cpu_idx].cap[cid] = sge->cap_states[idx].cap;
			break;
		}
	}

	mt_sched_printf(sched_eas_energy_calc,
		"cpu_idx=%d dst_cpu=%d cid=%d max_cpu=%d (util=%ld new=%ld) max_opp=%d (cap=%d)",
		cpu_idx, eenv->cpu[cpu_idx].cpu_id,
		cid, cpu, util, new_capacity,
		eenv->cpu[cpu_idx].cap_idx[cid],
		eenv->cpu[cpu_idx].cap[cid]);
}

#if defined(ARM_V8_2) && defined(CONFIG_MTK_UNIFY_POWER)
struct sched_group_energy cci_tbl;
const struct sched_group_energy * const cci_energy(void)
{
	struct sched_group_energy *sge = &cci_tbl;
	struct upower_tbl_info **addr_ptr_tbl_info;
	struct upower_tbl_info *ptr_tbl_info;
	struct upower_tbl *ptr_tbl;

	addr_ptr_tbl_info = upower_get_tbl();
	ptr_tbl_info = *addr_ptr_tbl_info;
	ptr_tbl = ptr_tbl_info[UPOWER_BANK_CCI].p_upower_tbl;

	sge->nr_cap_states = ptr_tbl->row_num;
	sge->cap_states = ptr_tbl->row;
	sge->lkg_idx = ptr_tbl->lkg_idx;
	return sge;
}

extern unsigned int mt_cpufreq_get_cur_cci_freq_idx(void);
void get_cci_volt(struct sg_state *cci)
{
	const struct sched_group_energy *_sge;
	static int CCI_nr_cap_stats;

	_sge = cci_energy();

	if (CCI_nr_cap_stats == 0) {
		CCI_nr_cap_stats = _sge->nr_cap_states;
	}

	cci->cap_idx = CCI_nr_cap_stats - mt_cpufreq_get_cur_cci_freq_idx();
	cci->volt = _sge->cap_states[cci->cap_idx].volt;
}
#else
void get_cci_volt(struct sg_state *cci)
{
}
#endif

void share_buck_volt(struct energy_env *eenv, int cpu_idx, int cid,
			struct sg_state *co_buck)
{
	if (is_share_buck(cid, &(co_buck->cid))) {
		int num_cluster = arch_get_nr_clusters();
		int cap_idx = eenv->cpu[cpu_idx].cap_idx[cid];

		if (co_buck->cid < num_cluster) {
			struct cpumask cls_cpus;
			const struct sched_group_energy *sge_core;
			int cpu;

			arch_get_cluster_cpus(&cls_cpus, co_buck->cid);
			cpu = cpumask_first(&cls_cpus);
			sge_core = cpu_core_energy(cpu);
			co_buck->cap_idx =
				eenv->cpu[cpu_idx].cap_idx[co_buck->cid];
			co_buck->volt =
				sge_core->cap_states[co_buck->cap_idx].volt;
#if defined(ARM_V8_2) && defined(CONFIG_MTK_UNIFY_POWER)
		} else if (co_buck->cid ==  CCI_ID) {    /* CCI + DSU */
			get_cci_volt(co_buck);
#endif
		}

		trace_sched_share_buck(cpu_idx, cid, cap_idx, co_buck->cid,
				co_buck->cap_idx, co_buck->volt);
	}
}

int
mtk_idle_power(int cpu_idx, int idle_state, int cpu, void *argu, int sd_level)
{
	struct energy_env *eenv = (struct energy_env *)argu;
	const struct sched_group_energy *_sge, *sge_core, *sge_clus;
	struct sched_domain *sd;
	unsigned long volt;
	int energy_cost = 0;
#ifdef CONFIG_ARM64
	int cid = cpu_topology[cpu].cluster_id;
#else
	int cid = cpu_topology[cpu].socket_id;
#endif
	int cap_idx = eenv->cpu[cpu_idx].cap_idx[cid];
	struct sg_state co_buck =  {-1, -1, 0};

	sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd);
	/* [FIXME] racing with hotplug */
	if (!sd)
		return 0;

	/* [FIXME] racing with hotplug */
	if (cap_idx == -1)
		return 0;

	_sge = cpu_core_energy(cpu);
	volt =  _sge->cap_states[cap_idx].volt;
	share_buck_volt(eenv, cpu_idx, cid, &co_buck);

	if (co_buck.volt > volt)
		cap_idx = share_buck_lkg_idx(_sge, cap_idx, co_buck.volt);

	_sge = sge_core = sge_clus = NULL;
	/* To handle only 1 CPU in cluster by HPS */
	if (unlikely(!sd->child &&
	   (rcu_dereference(per_cpu(sd_scs, cpu)) == NULL))) {
		struct upower_tbl_row *cpu_pwr_tbl, *clu_pwr_tbl;
		sge_core = cpu_core_energy(cpu);
		sge_clus = cpu_cluster_energy(cpu);

		cpu_pwr_tbl = &sge_core->cap_states[cap_idx];
		clu_pwr_tbl = &sge_clus->cap_states[cap_idx];

		/* idle: core->leask_power + cluster->lkg_pwr */
		energy_cost = cpu_pwr_tbl->lkg_pwr[sge_core->lkg_idx] +
				clu_pwr_tbl->lkg_pwr[sge_clus->lkg_idx];

		mt_sched_printf(sched_eas_energy_calc,
			"%s: %s lv=%d tlb_cpu[%d].leak=%d tlb_clu[%d].leak=%d total=%d",
			__func__, "WFI", sd_level,
			cap_idx,
			cpu_pwr_tbl->lkg_pwr[sge_core->lkg_idx],
			cap_idx,
			clu_pwr_tbl->lkg_pwr[sge_clus->lkg_idx],
			energy_cost);
	} else {
		struct upower_tbl_row *pwr_tbl;
		unsigned long lkg_pwr;

		if (sd_level == 0)
			_sge = cpu_core_energy(cpu); /* for cpu */
		else
			_sge = cpu_cluster_energy(cpu); /* for cluster */

		pwr_tbl =  &_sge->cap_states[cap_idx];
		lkg_pwr = pwr_tbl->lkg_pwr[_sge->lkg_idx];
		energy_cost = lkg_pwr;

		trace_sched_idle_power(sd_level, cap_idx, lkg_pwr, energy_cost);
	}

	idle_state = 0;

#if defined(ARM_V8_2) && defined(CONFIG_MTK_UNIFY_POWER)
	if ((sd_level != 0) && (co_buck.cid == CCI_ID)) {
		struct upower_tbl_row *CCI_pwr_tbl;
		unsigned long lkg_pwr;

		_sge = cci_energy();

		CCI_pwr_tbl = &_sge->cap_states[cap_idx];
		lkg_pwr = CCI_pwr_tbl->lkg_pwr[_sge->lkg_idx];
		energy_cost += lkg_pwr;
		trace_sched_idle_power(sd_level, cap_idx, lkg_pwr, energy_cost);
	}
#endif

	return energy_cost;
}

int calc_busy_power(const struct sched_group_energy *_sge, int cap_idx,
				unsigned long co_volt, int sd_level)
{
	unsigned long dyn_pwr, lkg_pwr;
	unsigned long volt;

	volt = _sge->cap_states[cap_idx].volt;
	calc_pwr(sd_level, _sge, cap_idx, volt, co_volt, &dyn_pwr, &lkg_pwr);

	return dyn_pwr + lkg_pwr;
}

int mtk_busy_power(int cpu_idx, int cpu, void *argu, int sd_level)
{
	struct energy_env *eenv = (struct energy_env *)argu;
	const struct sched_group_energy *_sge;
	struct sched_domain *sd;
	int energy_cost = 0;
#ifdef CONFIG_ARM64
	int cid = cpu_topology[cpu].cluster_id;
#else
	int cid = cpu_topology[cpu].socket_id;
#endif
	int cap_idx = eenv->cpu[cpu_idx].cap_idx[cid];
	struct sg_state co_buck = {-1, -1, 0};

	sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd);
	/* [FIXME] racing with hotplug */
	if (!sd)
		return 0;

	/* [FIXME] racing with hotplug */
	if (cap_idx == -1)
		return 0;

	share_buck_volt(eenv, cpu_idx, cid, &co_buck);
	/* To handle only 1 CPU in cluster by HPS */
	if (unlikely(!sd->child &&
		(rcu_dereference(per_cpu(sd_scs, cpu)) == NULL))) {
		/* fix HPS defeats: only one CPU in this cluster */

		_sge = cpu_core_energy(cpu); /* for CPU */
		energy_cost = calc_busy_power(_sge, cap_idx, co_buck.volt,
							0);
		_sge = cpu_cluster_energy(cpu); /* for cluster */
		energy_cost += calc_busy_power(_sge, cap_idx, co_buck.volt,
							1);
	} else {
		if (sd_level == 0)
			_sge = cpu_core_energy(cpu); /* for CPU */
		else
			_sge = cpu_cluster_energy(cpu); /* for cluster */

		energy_cost = calc_busy_power(_sge, cap_idx, co_buck.volt,
							sd_level);
	}

#if defined(ARM_V8_2) && defined(CONFIG_MTK_UNIFY_POWER)
	if ((sd_level != 0) && (co_buck.cid == CCI_ID)) {
		/* CCI + DSU */
		unsigned long volt;

		_sge = cpu_core_energy(cpu); /* for CPU */
		volt =  _sge->cap_states[cap_idx].volt;

		_sge = cci_energy();
		energy_cost += calc_busy_power(_sge, co_buck.cap_idx, volt,
							sd_level);
	}
#endif

	return energy_cost;
}
#endif

#ifdef CONFIG_MTK_SCHED_EAS_POWER_SUPPORT
void mtk_update_new_capacity(struct energy_env *eenv)
{
	int i, cpu_idx;

	/* To get max opp index of every cluster for power estimation of
	 * share buck
	 */
	for (cpu_idx = EAS_CPU_PRV; cpu_idx < eenv->max_cpu_count ; ++cpu_idx) {
		if (eenv->cpu[cpu_idx].cpu_id == -1)
			continue;

		for (i = 0; i < arch_get_nr_clusters(); i++)
			mtk_cluster_capacity_idx(i, eenv, cpu_idx);
	}

}
#else
void mtk_update_new_capacity(struct energy_env *eenv)
{
}

static int __find_energy_efficient_cpu(struct sched_domain *sd,
				     struct task_struct *p,
				     int cpu, int prev_cpu,
				     int sync)
{
	return find_energy_efficient_cpu(sd, p, cpu, prev_cpu, sync);
}
#endif

#ifdef CONFIG_MTK_SCHED_BOOST
static void select_task_prefer_cpu_fair(struct task_struct *p, int *result)
{
	int task_prefer;
	int cpu, new_cpu;

	task_prefer = cpu_prefer(p);

	cpu = (*result & LB_CPU_MASK);

	new_cpu = select_task_prefer_cpu(p, cpu);

	if ((new_cpu >= 0)  && (new_cpu != cpu)) {
		if (task_prefer_match(p, cpu))
			*result = new_cpu | LB_THERMAL;
		else
			*result = new_cpu | LB_HINT;
	}
}

#else

static void select_task_prefer_cpu_fair(struct task_struct *p, int *result)
{
}

#endif

inline int
task_match_on_dst_cpu(struct task_struct *p, int src_cpu, int target_cpu)
{
	struct task_struct *target_tsk;
	struct rq *rq = cpu_rq(target_cpu);

#ifdef CONFIG_MTK_SCHED_BOOST
	if (task_prefer_match(p, src_cpu))
		return 0;

	target_tsk = rq->curr;
	if (task_prefer_fit(target_tsk, target_cpu))
		return 0;
#endif

	return 1;
}

static int check_freq_turning(void)
{
	struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
	unsigned long capacity_curr_little, capacity_curr_big;

	if (rd->min_cap_orig_cpu < 0 || rd->max_cap_orig_cpu < 0)
		return false;

	capacity_curr_little = capacity_curr_of(rd->min_cap_orig_cpu);
	capacity_curr_big = capacity_curr_of(rd->max_cap_orig_cpu);

	if ((capacity_curr_little > cpu_eff_tp) &&
			(capacity_curr_big <=  big_cpu_eff_tp))
		return true;

	return false;
}

struct task_rotate_work {
	struct work_struct w;
	struct task_struct *src_task;
	struct task_struct *dst_task;
	int src_cpu;
	int dst_cpu;
};

static DEFINE_PER_CPU(struct task_rotate_work, task_rotate_works);
struct task_rotate_reset_uclamp_work task_rotate_reset_uclamp_works;
unsigned int sysctl_sched_rotation_enable;
bool set_uclamp;

void set_sched_rotation_enable(bool enable)
{
	sysctl_sched_rotation_enable = enable;
}

bool is_min_capacity_cpu(int cpu)
{
	struct root_domain *rd = cpu_rq(smp_processor_id())->rd;

	if (rd->min_cap_orig_cpu < 0)
		return false;

	if (capacity_orig_of(cpu) == capacity_orig_of(rd->min_cap_orig_cpu))
		return true;

	return false;
}

static void task_rotate_work_func(struct work_struct *work)
{
	struct task_rotate_work *wr = container_of(work,
				struct task_rotate_work, w);
	int ret = -1;
	struct rq *src_rq, *dst_rq;

	ret = migrate_swap(wr->src_task, wr->dst_task);

	if (ret == 0) {
		update_eas_uclamp_min(EAS_UCLAMP_KIR_BIG_TASK, CGROUP_TA,
				scale_to_percent(SCHED_CAPACITY_SCALE));
		set_uclamp = true;
		trace_sched_big_task_rotation(wr->src_cpu, wr->dst_cpu,
						wr->src_task->pid,
						wr->dst_task->pid,
						true, set_uclamp);
	}

	put_task_struct(wr->src_task);
	put_task_struct(wr->dst_task);

	src_rq = cpu_rq(wr->src_cpu);
	dst_rq = cpu_rq(wr->dst_cpu);

	local_irq_disable();
	double_rq_lock(src_rq, dst_rq);
	src_rq->active_balance = 0;
	dst_rq->active_balance = 0;
	double_rq_unlock(src_rq, dst_rq);
	local_irq_enable();
}

static void task_rotate_reset_uclamp_work_func(struct work_struct *work)
{
	update_eas_uclamp_min(EAS_UCLAMP_KIR_BIG_TASK, CGROUP_TA, 0);
	set_uclamp = false;
	trace_sched_big_task_rotation_reset(set_uclamp);
}

void task_rotate_work_init(void)
{
	int i;

	for_each_possible_cpu(i) {
		struct task_rotate_work *wr = &per_cpu(task_rotate_works, i);

		INIT_WORK(&wr->w, task_rotate_work_func);
	}

	INIT_WORK(&task_rotate_reset_uclamp_works.w,
			task_rotate_reset_uclamp_work_func);
}

void task_check_for_rotation(struct rq *src_rq)
{
	u64 wc, wait, max_wait = 0, run, max_run = 0;
	int deserved_cpu = nr_cpu_ids, dst_cpu = nr_cpu_ids;
	int i, src_cpu = cpu_of(src_rq);
	struct rq *dst_rq;
	struct task_rotate_work *wr = NULL;
	int heavy_task = 0;
	int force = 0;

	if (!sysctl_sched_rotation_enable)
		return;

	if (is_max_capacity_cpu(src_cpu))
		return;

	for_each_possible_cpu(i) {
		struct rq *rq = cpu_rq(i);
		struct task_struct *curr_task = rq->curr;

		if (curr_task &&
			!task_fits_capacity(curr_task, capacity_of(i)))
			heavy_task += 1;
	}

	if (heavy_task < HEAVY_TASK_NUM)
		return;

	wc = ktime_get_ns();
	for_each_possible_cpu(i) {
		struct rq *rq = cpu_rq(i);

		if (!is_min_capacity_cpu(i))
			continue;

		if (is_reserved(i))
			continue;

		if (!rq->misfit_task_load || rq->curr->sched_class !=
						&fair_sched_class)
			continue;

		wait = wc - rq->curr->last_enqueued_ts;
		if (wait > max_wait) {
			max_wait = wait;
			deserved_cpu = i;
		}
	}

	if (deserved_cpu != src_cpu)
		return;

	for_each_possible_cpu(i) {
		struct rq *rq = cpu_rq(i);

		if (capacity_orig_of(i) <= capacity_orig_of(src_cpu))
			continue;

		if (is_reserved(i))
			continue;

		if (rq->curr->sched_class != &fair_sched_class)
			continue;

		if (rq->nr_running > 1)
			continue;

		run = wc - rq->curr->last_enqueued_ts;

		if (run < TASK_ROTATION_THRESHOLD_NS)
			continue;

		if (run > max_run) {
			max_run = run;
			dst_cpu = i;
		}
	}

	if (dst_cpu == nr_cpu_ids)
		return;

	dst_rq = cpu_rq(dst_cpu);

	double_rq_lock(src_rq, dst_rq);
	if (dst_rq->curr->sched_class == &fair_sched_class) {

		if (!cpumask_test_cpu(dst_cpu,
					&(src_rq->curr)->cpus_allowed) ||
			!cpumask_test_cpu(src_cpu,
					&(dst_rq->curr)->cpus_allowed)) {
			double_rq_unlock(src_rq, dst_rq);
			return;
		}

		if (!src_rq->active_balance && !dst_rq->active_balance) {
			src_rq->active_balance = MIGR_ROTATION;
			dst_rq->active_balance = MIGR_ROTATION;

			get_task_struct(src_rq->curr);
			get_task_struct(dst_rq->curr);

			wr = &per_cpu(task_rotate_works, src_cpu);

			wr->src_task = src_rq->curr;
			wr->dst_task = dst_rq->curr;

			wr->src_cpu = src_rq->cpu;
			wr->dst_cpu = dst_rq->cpu;
			force = 1;
		}
	}
	double_rq_unlock(src_rq, dst_rq);

	if (force) {
		queue_work_on(src_cpu, system_highpri_wq, &wr->w);
		trace_sched_big_task_rotation(wr->src_cpu, wr->dst_cpu,
					wr->src_task->pid, wr->dst_task->pid,
					false, set_uclamp);
	}
}