/* * Copyright (C) 2017 MediaTek Inc. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See http://www.gnu.org/licenses/gpl-2.0.html for more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // TODO: remove comment after met ready //#include #include #include "rq_stats.h" #define MAX_LONG_SIZE 24 #define DEFAULT_RQ_POLL_JIFFIES 1 #define DEFAULT_DEF_TIMER_JIFFIES 5 #define HEAVY_TASK_ENABLE 1 #ifdef CONFIG_MTK_SCHED_RQAVG_US struct rq_data rq_info; spinlock_t rqavg_lock; #endif /* set to 1 if per cpu load is cpu freq. variant */ static int cpufreq_variant = 1; #ifdef CONFIG_CPU_FREQ struct notifier_block freq_transition; #endif /* CONFIG_CPU_FREQ */ struct notifier_block cpu_hotplug; /* max=1023, for last_poll, threshold */ static unsigned int heavy_task_threshold = 920; /* max=1023 for AHT, threshold */ static unsigned int heavy_task_threshold2 = 920; /* max=99 for AHT, admission control */ static unsigned int avg_heavy_task_threshold = 65; static int htask_cpucap_ctrl = 1; #ifdef CONFIG_MTK_CORE_CTL #define OVERUTIL_THRESHOLD_SIZE 2 /* cap(LL)*0.8, cap(BL)*0.7 */ static unsigned int overutil_thres[OVERUTIL_THRESHOLD_SIZE] = {19, 60}; #else /* max = 100, threshold for capacity overutiled */ static int overutil_threshold = 35; #endif struct cpu_load_data { u64 prev_cpu_idle; u64 prev_cpu_wall; u64 prev_cpu_iowait; unsigned int avg_load_maxfreq_rel; unsigned int avg_load_maxfreq_abs; unsigned int samples; unsigned int window_size; unsigned int cur_freq; unsigned int policy_max; ktime_t last_update; cpumask_var_t related_cpus; spinlock_t cpu_load_lock; }; static DEFINE_PER_CPU(struct cpu_load_data, cpuload); //static DEFINE_PER_CPU(struct cpufreq_policy, cpupolicy); enum AVG_LOAD_ID { AVG_LOAD_IGNORE, AVG_LOAD_UPDATE }; #ifdef CONFIG_MTK_CORE_CTL unsigned int get_overutil_threshold(int index) { if (index >= 0 && index < OVERUTIL_THRESHOLD_SIZE) return overutil_thres[index]; return 100; } EXPORT_SYMBOL(get_overutil_threshold); #else int get_overutil_threshold(void) { return overutil_threshold; } #endif #ifdef CONFIG_SCHED_HMP_PRIO_FILTER static unsigned int heavy_task_prio = NICE_TO_PRIO(CONFIG_SCHED_HMP_PRIO_FILTER_VAL); #define task_low_priority(prio) ((prio >= heavy_task_prio)?1:0) #endif #ifdef CONFIG_CPU_FREQ #include #else /* !CONFIG_CPU_FREQ */ static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) { u64 idle_time; u64 cur_wall_time; u64 busy_time; cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; idle_time = cur_wall_time - busy_time; if (wall) *wall = cputime_to_usecs(cur_wall_time); return cputime_to_usecs(idle_time); } static inline u64 get_cpu_idle_time_internal(unsigned int cpu, u64 *wall) { u64 idle_time = get_cpu_idle_time_us(cpu, NULL); if (idle_time == -1ULL) idle_time = get_cpu_idle_time_jiffy(cpu, wall); else idle_time += get_cpu_iowait_time_us(cpu, wall); return idle_time; } #endif /* CONFIG_CPU_FREQ */ static inline u64 get_cpu_iowait_time(unsigned int cpu, u64 *wall) { u64 iowait_time = get_cpu_iowait_time_us(cpu, wall); if (iowait_time == -1ULL) return 0; return iowait_time; } static int update_average_load(enum AVG_LOAD_ID id, unsigned int freq, unsigned int cpu) { struct cpu_load_data *pcpu = &per_cpu(cpuload, cpu); u64 cur_wall_time, cur_idle_time, cur_iowait_time; unsigned int idle_time, wall_time, iowait_time; unsigned int cur_load, prev_avg_load_rel, prev_avg_load_abs; unsigned int load_at_max_freq_rel, load_at_max_freq_abs; u64 prev_wall_time, prev_cpu_idle, prev_cpu_iowait; #ifdef CONFIG_CPU_FREQ cur_idle_time = get_cpu_idle_time(cpu, &cur_wall_time, 0); #else /* !CONFIG_CPU_FREQ */ cur_idle_time = get_cpu_idle_time_internal(cpu, &cur_wall_time); #endif /* CONFIG_CPU_FREQ */ cur_iowait_time = get_cpu_iowait_time(cpu, &cur_wall_time); prev_wall_time = pcpu->prev_cpu_wall; prev_cpu_idle = pcpu->prev_cpu_idle; prev_cpu_iowait = pcpu->prev_cpu_iowait; if (unlikely(cur_wall_time < pcpu->prev_cpu_wall)) return 0; wall_time = (unsigned int) (cur_wall_time - pcpu->prev_cpu_wall); pcpu->prev_cpu_wall = cur_wall_time; idle_time = (unsigned int) (cur_idle_time - pcpu->prev_cpu_idle); pcpu->prev_cpu_idle = cur_idle_time; iowait_time = (unsigned int) (cur_iowait_time - pcpu->prev_cpu_iowait); pcpu->prev_cpu_iowait = cur_iowait_time; if (idle_time >= iowait_time) idle_time -= iowait_time; if (unlikely(!wall_time || wall_time < idle_time)) return 0; if (id == AVG_LOAD_IGNORE) cur_load = 0; else cur_load = 100 * (wall_time - idle_time) / wall_time; /* Calculate the scaled load across CPU */ if (cpu_online(cpu)) { load_at_max_freq_abs = (cur_load * freq) / pcpu->policy_max; load_at_max_freq_rel = cur_load; } else { load_at_max_freq_abs = 0; load_at_max_freq_rel = 0; } prev_avg_load_rel = pcpu->avg_load_maxfreq_rel; prev_avg_load_abs = pcpu->avg_load_maxfreq_abs; #if 1 if (!pcpu->avg_load_maxfreq_rel || !pcpu->avg_load_maxfreq_abs) { /* This is the first sample in this window*/ pcpu->avg_load_maxfreq_rel = load_at_max_freq_rel; pcpu->avg_load_maxfreq_abs = load_at_max_freq_abs; pcpu->window_size = wall_time; } else { /* * The is already a sample available in this window. * Compute weighted average with prev entry, so that we get * the precise weighted load. */ pcpu->avg_load_maxfreq_rel = ((pcpu->avg_load_maxfreq_rel * pcpu->window_size) + (load_at_max_freq_rel * wall_time)) / (wall_time + pcpu->window_size); pcpu->avg_load_maxfreq_abs = ((pcpu->avg_load_maxfreq_abs * pcpu->window_size) + (load_at_max_freq_abs * wall_time)) / (wall_time + pcpu->window_size); pcpu->window_size += wall_time; } #else /* debug only */ pcpu->avg_load_maxfreq_rel = load_at_max_freq_rel; pcpu->avg_load_maxfreq_abs = load_at_max_freq_abs; pcpu->window_size = wall_time; #endif return 0; } #ifdef CONFIG_MTK_SCHED_CPULOAD static void get_percpu_load (int cpu, bool reset, unsigned int *rel_load, unsigned int *abs_load) { struct cpu_load_data *pcpu; unsigned long flags; if (!rel_load || !abs_load) return; *rel_load = 0; *abs_load = 0; if (rq_info.init != 1) { *rel_load = 90; *abs_load = 90; return; } pcpu = &per_cpu(cpuload, cpu); spin_lock_irqsave(&pcpu->cpu_load_lock, flags); update_average_load(AVG_LOAD_UPDATE, pcpu->cur_freq, cpu); *rel_load = pcpu->avg_load_maxfreq_rel; *abs_load = pcpu->avg_load_maxfreq_abs; if (reset) { pcpu->avg_load_maxfreq_rel = 0; pcpu->avg_load_maxfreq_abs = 0; } pcpu->last_update = ktime_get(); spin_unlock_irqrestore(&pcpu->cpu_load_lock, flags); } struct cpu_loading { int rel_load; int abs_load; }; static DEFINE_PER_CPU(struct cpu_loading, cpuloading); // TODO: patch back met_cpu_load after MET ready /* to calculate cpu loading */ void cal_cpu_load(int cpu) { int rel_load, abs_load; int reset = 1; struct cpu_loading *this_cpu = &per_cpu(cpuloading, cpu); struct cpu_load_data *pcpu = &per_cpu(cpuload, cpu); ktime_t now = ktime_get(); /* periodic: 20ms */ if (ktime_before(now, ktime_add_ms(pcpu->last_update, (CPU_LOAD_AVG_DEFAULT_MS - CPU_LOAD_AVG_TOLERANCE)))) return; get_percpu_load(cpu, reset, &rel_load, &abs_load); this_cpu->rel_load = rel_load; this_cpu->abs_load = abs_load; // TODO: remove comment after met ready //met_tag_oneshot(0, met_cpu_load[cpu], sched_get_cpu_load(cpu)); } /* Legacy interfaces to export */ void sched_get_percpu_load2(int cpu, bool reset, unsigned int *rel_load, unsigned int *abs_load) { struct cpu_loading *this_cpu = &per_cpu(cpuloading, cpu); struct cpu_load_data *pcpu = &per_cpu(cpuload, cpu); ktime_t now = ktime_get(); if (!rel_load || !abs_load) return; *rel_load = 0; *abs_load = 0; if (rq_info.init != 1) { *rel_load = 90; *abs_load = 90; return; } /* * If cpu loading was last updated long enough, * don't take care this loading into account. */ if (ktime_after(now, ktime_add_ms(pcpu->last_update, (CPU_LOAD_AVG_DEFAULT_MS*2)))) { *rel_load = 0; *abs_load = 0; return; } *rel_load = this_cpu->rel_load; *abs_load = this_cpu->abs_load; } EXPORT_SYMBOL(sched_get_percpu_load2); /* Legacy interfaces to export */ unsigned int sched_get_percpu_load(int cpu, bool reset, bool use_maxfreq) { struct cpu_loading *this_cpu = &per_cpu(cpuloading, cpu); struct cpu_load_data *pcpu = &per_cpu(cpuload, cpu); ktime_t now = ktime_get(); if (rq_info.init != 1) return 90; /* If cpu loading was last updated long enough, ignore it. */ if (ktime_after(now, ktime_add_ms(pcpu->last_update, (CPU_LOAD_AVG_DEFAULT_MS*2)))) return 0; return use_maxfreq ? this_cpu->abs_load : this_cpu->rel_load; } EXPORT_SYMBOL(sched_get_percpu_load); /* New interface to export */ unsigned int sched_get_cpu_load(int cpu) { struct cpu_loading *this_cpu = &per_cpu(cpuloading, cpu); struct cpu_load_data *pcpu = &per_cpu(cpuload, cpu); ktime_t now = ktime_get(); if (rq_info.init != 1) return 90; /* If cpu loading was last updated long enough, ignore it. */ if (ktime_after(now, ktime_add_ms(pcpu->last_update, (CPU_LOAD_AVG_DEFAULT_MS*2)))) return 0; return this_cpu->rel_load; } EXPORT_SYMBOL(sched_get_cpu_load); #else void sched_get_percpu_load2(int cpu, bool reset, unsigned int *rel_load, unsigned int *abs_load) { struct cpu_load_data *pcpu; unsigned long flags; if (!rel_load || !abs_load) return; *rel_load = 0; *abs_load = 0; if (rq_info.init != 1) { *rel_load = 90; *abs_load = 90; return; } pcpu = &per_cpu(cpuload, cpu); spin_lock_irqsave(&pcpu->cpu_load_lock, flags); update_average_load(AVG_LOAD_UPDATE, pcpu->cur_freq, cpu); *rel_load = pcpu->avg_load_maxfreq_rel; *abs_load = pcpu->avg_load_maxfreq_abs; if (reset) { pcpu->avg_load_maxfreq_rel = 0; pcpu->avg_load_maxfreq_abs = 0; } spin_unlock_irqrestore(&pcpu->cpu_load_lock, flags); } EXPORT_SYMBOL(sched_get_percpu_load2); unsigned int sched_get_percpu_load(int cpu, bool reset, bool use_maxfreq) { int rel_load, abs_load; sched_get_percpu_load2(cpu, reset, &rel_load, &abs_load); return use_maxfreq ? abs_load : rel_load; } EXPORT_SYMBOL(sched_get_percpu_load); #endif /* #define DETECT_HTASK_HEAT */ #ifdef DETECT_HTASK_HEAT #define MAX_HTASK_TEMPERATURE 10 static unsigned int htask_temperature; static void __heat_refined(int *count) { if (!arch_is_smp()) { if (*count) { htask_temperature += (htask_temperature < MAX_HTASK_TEMPERATURE) ? 1 : 0; } else { *count = (htask_temperature > 0) ? 1 : 0; htask_temperature -= (htask_temperature > 0) ? 1 : 0; } } } #else static inline void __heat_refined(int *count) {} #endif static void __trace_out(int heavy, int cpu, struct task_struct *p) { #define TRACEBUF_LEN 128 char tracebuf[TRACEBUF_LEN]; snprintf(tracebuf, TRACEBUF_LEN, "[hvytask_poll] %s cpu=%d load=%4lu cpucap=%4lu/%4lu pid=%4d name=%s", heavy ? "Y" : "N", cpu, p->se.avg.load_avg, topology_cur_cpu_capacity(cpu), arch_scale_cpu_capacity(NULL, cpu), p->pid, p->comm); trace_sched_heavy_task(tracebuf); } static int ack_by_curcap(int cpu, int cluster_id, int max_cluster_id) { unsigned long cur_cap; unsigned long max_cap; int acked = 0; int thrshld; if (!htask_cpucap_ctrl) /* disable cpucap control */ return 1; if (cluster_id == max_cluster_id) return 1; /* skip checking for the fastest cluster */ else if (cluster_id == 0) thrshld = SCHED_CAPACITY_SCALE*60/100; else thrshld = SCHED_CAPACITY_SCALE*40/100; cur_cap = topology_cur_cpu_capacity(cpu); max_cap = arch_scale_cpu_capacity(NULL, cpu); if ((cur_cap * SCHED_CAPACITY_SCALE) >= max_cap * thrshld) acked = 1; else acked = 0; return acked; } int is_ack_curcap(int cpu) { int cluster_id, cluster_nr; cluster_nr = arch_get_nr_clusters(); cluster_id = arch_get_cluster_id(cpu); return ack_by_curcap(cpu, cluster_id, cluster_nr-1); } EXPORT_SYMBOL(is_ack_curcap); int is_heavy_task(struct task_struct *p) { if (!HEAVY_TASK_ENABLE) return 0; if (!p) return 0; #ifdef CONFIG_SCHED_HMP_PRIO_FILTER if (task_low_priority(p->prio)) return 0; #endif if (p->se.avg.loadwop_avg >= heavy_task_threshold2) return 1; return 0; } EXPORT_SYMBOL(is_heavy_task); int inc_nr_heavy_running(int invoker, struct task_struct *p, int inc, bool ack_cap) { #ifdef CONFIG_MTK_SCHED_RQAVG_KS sched_update_nr_heavy_prod(invoker, p, cpu_of(task_rq(p)), inc, ack_cap); #endif return 0; } EXPORT_SYMBOL(inc_nr_heavy_running); static unsigned int htask_statistic; unsigned int sched_get_nr_heavy_task_by_threshold(int cluster_id, unsigned int threshold) { int cpu; struct task_struct *p; unsigned long flags; unsigned int count = 0; int is_heavy = 0; int clusters; struct cpumask cls_cpus; if (rq_info.init != 1) return 0; clusters = arch_get_nr_clusters(); if (cluster_id < 0 || cluster_id >= clusters) { printk_deferred("[%s] invalid cluster id %d\n", __func__, cluster_id); return 0; } arch_get_cluster_cpus(&cls_cpus, cluster_id); for_each_cpu(cpu, &cls_cpus) { if (likely(!cpu_online(cpu))) continue; raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); list_for_each_entry(p, &cpu_rq(cpu)->cfs_tasks, se.group_node) { is_heavy = 0; #ifdef CONFIG_SCHED_HMP_PRIO_FILTER if (task_low_priority(p->prio)) continue; #endif if (p->se.avg.loadwop_avg >= threshold) { is_heavy = ack_by_curcap(cpu, cluster_id, clusters-1); count += is_heavy ? 1 : 0; __trace_out(is_heavy, cpu, p); } } raw_spin_unlock_irqrestore(&cpu_rq(cpu)->lock, flags); } __heat_refined(&count); if (count) htask_statistic++; return count; } EXPORT_SYMBOL(sched_get_nr_heavy_task_by_threshold); unsigned int sched_get_nr_heavy_task(void) { int nr_clusters = arch_get_nr_clusters(); int id, total_htask = 0; for (id = 0; id < nr_clusters; id++) total_htask += sched_get_nr_heavy_task_by_threshold(id, heavy_task_threshold); return total_htask; } EXPORT_SYMBOL(sched_get_nr_heavy_task); /* sched_get_nr_heavy_task2: * return max heavy tasks nr * in the cluster between last_poll * and average heavy task * if CONFIG_MTK_SCHED_RQAVG_KS is defined. */ unsigned int sched_get_nr_heavy_task2(int cluster_id) { int lastpoll_htask1 = 0, lastpoll_htask2 = 0; int avg_htask = 0, avg_htask_scal = 0; int max; lastpoll_htask1 = sched_get_nr_heavy_task_by_threshold(cluster_id, heavy_task_threshold); #ifdef CONFIG_MTK_SCHED_RQAVG_KS lastpoll_htask2 = sched_get_nr_heavy_running_avg(cluster_id, &avg_htask_scal); #endif avg_htask = (avg_htask_scal%100 >= avg_heavy_task_threshold) ? (avg_htask_scal/100+1):(avg_htask_scal/100); max = max(max(lastpoll_htask1, lastpoll_htask2), avg_htask); trace_sched_avg_heavy_task(lastpoll_htask1, lastpoll_htask2, avg_htask_scal, cluster_id, max); return max; } EXPORT_SYMBOL(sched_get_nr_heavy_task2); void sched_set_heavy_task_threshold(unsigned int val) { heavy_task_threshold = val; } EXPORT_SYMBOL(sched_set_heavy_task_threshold); #if 0 static int cpufreq_policy_handler(struct notifier_block *nb, unsigned long event, void *data) { int cpu = 0; struct cpufreq_policy *policy = data; struct cpu_load_data *this_cpu; if (event == CPUFREQ_START) return 0; if (event != CPUFREQ_INCOMPATIBLE) return 0; /* Make sure that all CPUs impacted by this policy are * updated since we will only get a notification when the * user explicitly changes the policy on a CPU. */ for_each_cpu(cpu, policy->cpus) { struct cpu_load_data *pcpu = &per_cpu(cpuload, j); spin_lock_irqsave(&pcpu->cpu_load_lock, flags); pcpu->policy_max = policy->cpuinfo.max_freq; spin_unlock_irqrestore(&pcpu->cpu_load_lock, flags); } return 0; } #endif #ifdef CONFIG_CPU_FREQ static int cpufreq_transition_handler(struct notifier_block *nb, unsigned long val, void *data) { struct cpufreq_freqs *freqs = data; struct cpu_load_data *this_cpu = &per_cpu(cpuload, freqs->cpu); int j; unsigned long flags; if (rq_info.init != 1) return 0; switch (val) { case CPUFREQ_POSTCHANGE: for_each_cpu(j, this_cpu->related_cpus) { struct cpu_load_data *pcpu = &per_cpu(cpuload, j); /* flush previous laod */ if (pcpu->policy_max == 1) { struct cpufreq_policy cpu_policy; if (!cpufreq_get_policy(&cpu_policy, j)) { spin_lock_irqsave(&pcpu->cpu_load_lock, flags); pcpu->policy_max = cpu_policy.cpuinfo.max_freq; spin_unlock_irqrestore (&pcpu->cpu_load_lock, flags); } } spin_lock_irqsave(&pcpu->cpu_load_lock, flags); if (cpu_online(j)) update_average_load(AVG_LOAD_UPDATE, freqs->old, freqs->cpu); pcpu->cur_freq = cpufreq_variant ? freqs->new : pcpu->policy_max; spin_unlock_irqrestore(&pcpu->cpu_load_lock, flags); } break; } return 0; } #endif /* CONFIG_CPU_FREQ */ static int system_suspend_handler(struct notifier_block *nb, unsigned long val, void *data) { switch (val) { case PM_POST_HIBERNATION: case PM_POST_SUSPEND: case PM_POST_RESTORE: rq_info.hotplug_disabled = 0; break; case PM_HIBERNATION_PREPARE: case PM_SUSPEND_PREPARE: rq_info.hotplug_disabled = 1; break; default: return NOTIFY_DONE; } return NOTIFY_OK; } int get_avg_heavy_task_threshold(void) { return avg_heavy_task_threshold; } int get_heavy_task_threshold(void) { return heavy_task_threshold; } static ssize_t hotplug_disable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int val = 0; val = rq_info.hotplug_disabled; return snprintf(buf, MAX_LONG_SIZE, "%d\n", val); } static struct kobj_attribute hotplug_disabled_attr = __ATTR_RO(hotplug_disable); #ifdef CONFIG_MTK_SCHED_RQAVG_US_ENABLE_WQ static void def_work_fn(struct work_struct *work) { int64_t diff; diff = ktime_to_ns(ktime_get()) - rq_info.def_start_time; do_div(diff, 1000 * 1000); rq_info.def_interval = (unsigned int) diff; /* Notify polling threads on change of value */ sysfs_notify(rq_info.kobj, NULL, "def_timer_ms"); } #endif static ssize_t run_queue_avg_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int val = 0; unsigned long flags = 0; spin_lock_irqsave(&rqavg_lock, flags); /* rq avg currently available only on one core */ val = rq_info.rq_avg; rq_info.rq_avg = 0; spin_unlock_irqrestore(&rqavg_lock, flags); return snprintf(buf, PAGE_SIZE, "%d.%d\n", val/10, val%10); } static struct kobj_attribute run_queue_avg_attr = __ATTR_RO(run_queue_avg); static ssize_t show_run_queue_poll_ms(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { int ret = 0; unsigned long flags = 0; spin_lock_irqsave(&rqavg_lock, flags); ret = snprintf(buf, MAX_LONG_SIZE, "%u\n", jiffies_to_msecs(rq_info.rq_poll_jiffies)); spin_unlock_irqrestore(&rqavg_lock, flags); return ret; } static ssize_t store_run_queue_poll_ms(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; unsigned long flags = 0; static DEFINE_MUTEX(lock_poll_ms); mutex_lock(&lock_poll_ms); spin_lock_irqsave(&rqavg_lock, flags); if (sscanf(buf, "%iu", &val) != 0) rq_info.rq_poll_jiffies = msecs_to_jiffies(val); spin_unlock_irqrestore(&rqavg_lock, flags); mutex_unlock(&lock_poll_ms); return count; } static struct kobj_attribute run_queue_poll_ms_attr = __ATTR(run_queue_poll_ms, 0600 /* S_IWUSR | S_IRUSR */, show_run_queue_poll_ms, store_run_queue_poll_ms); static ssize_t show_def_timer_ms(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return snprintf(buf, MAX_LONG_SIZE, "%u\n", rq_info.def_interval); } static ssize_t store_def_timer_ms(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; if (sscanf(buf, "%iu", &val) != 0) rq_info.def_timer_jiffies = msecs_to_jiffies(val); rq_info.def_start_time = ktime_to_ns(ktime_get()); return count; } static struct kobj_attribute def_timer_ms_attr = __ATTR(def_timer_ms, 0600 /* S_IWUSR | S_IRUSR */, show_def_timer_ms, store_def_timer_ms); static ssize_t cpu_normalized_load_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { int cpu; unsigned int len = 0; unsigned int load_rel = 0, load_abs = 0; unsigned int max_len = 4096; int i; for_each_possible_cpu(cpu) { sched_get_percpu_load2(cpu, 0, &load_rel, &load_abs); len += snprintf(buf+len, max_len-len, "cpu(%d): load(rel/abs) = %u/%u\n", cpu, load_rel, load_abs); } len += snprintf(buf+len, max_len-len, "htask_threshold=%d, total_htask#=%d\n", heavy_task_threshold, htask_statistic); for (i = 0; i < arch_get_nr_clusters(); i++) len += snprintf(buf+len, max_len-len, "cluster%d: current_htask#=%u\n", i, sched_get_nr_heavy_task2(i)); return len; } static struct kobj_attribute cpu_normalized_load_attr = __ATTR_RO(cpu_normalized_load); /* For htasks statistics */ static ssize_t show_heavy_tasks(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int len = 0; unsigned int max_len = 4096; int i; len += snprintf(buf+len, max_len-len, "htask_cpucap_ctrl=%d, htask_threshold=%d, total_htask#=%d\n", htask_cpucap_ctrl, heavy_task_threshold, htask_statistic); for (i = 0; i < arch_get_nr_clusters(); i++) len += snprintf(buf+len, max_len-len, "cluster%d: current_htask#=%u\n", i, sched_get_nr_heavy_task2(i)); return len; } static struct kobj_attribute htasks_attr = __ATTR(htasks, 0400 /* S_IRUSR */, show_heavy_tasks, NULL); /* For cpu capacity control */ static ssize_t htask_cpucap_ctrl_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; if (sscanf(buf, "%iu", &val) != 0) htask_cpucap_ctrl = (val == 0 ? 0 : 1); return count; } static struct kobj_attribute htask_cpucap_ctrl_attr = __ATTR_WO(htask_cpucap_ctrl); /* For read/write heavy_task_threshold */ static ssize_t store_heavy_task_threshold(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; if (sscanf(buf, "%iu", &val) != 0) sched_set_heavy_task_threshold(val); return count; } static ssize_t show_heavy_task_threshold(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int len = 0; unsigned int max_len = 4096; len += snprintf(buf, max_len, "%d\n", heavy_task_threshold); return len; } static struct kobj_attribute htasks_thresh_attr = __ATTR(htasks_thresh, 0600 /* S_IWUSR | S_IRUSR */, show_heavy_task_threshold, store_heavy_task_threshold); /* For read/write admission control for average heavy task */ static ssize_t store_avg_heavy_task_ac(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; if (sscanf(buf, "%iu", &val) != 0) { if (val >= 0 && val < 100) { avg_heavy_task_threshold = val; #ifdef CONFIG_MTK_SCHED_RQAVG_KS heavy_thresh_chg_notify(); #endif } } return count; } static ssize_t show_avg_heavy_task_ac(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int len = 0; unsigned int max_len = 4096; len += snprintf(buf, max_len, "%d\n", avg_heavy_task_threshold); return len; } static struct kobj_attribute avg_htasks_ac_attr = __ATTR(avg_htasks_ac, 0600 /* S_IWUSR | S_IRUSR */, show_avg_heavy_task_ac, store_avg_heavy_task_ac); #ifdef CONFIG_MTK_CORE_CTL void set_overutil_threshold(unsigned int index, unsigned int val) { if (index >= OVERUTIL_THRESHOLD_SIZE) return; if (val <= 100) { overutil_thres[index] = (int)val; #ifdef CONFIG_MTK_SCHED_RQAVG_KS overutil_thresh_chg_notify(); #endif } } EXPORT_SYMBOL(set_overutil_threshold); /* For read/write utilization related settings */ static ssize_t store_overutil(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val[OVERUTIL_THRESHOLD_SIZE]; int ret, i; ret = sscanf(buf, "%u %u\n", &val[0], &val[1]); if (ret <= 0 || OVERUTIL_THRESHOLD_SIZE <= ret) return -EINVAL; if (ret == 1) { for (i = 0; i < OVERUTIL_THRESHOLD_SIZE; i++) overutil_thres[i] = val[0] < 100 ? val[0] : overutil_thres[i]; } else { for (i = 0; i < OVERUTIL_THRESHOLD_SIZE; i++) overutil_thres[i] = val[i] < 100 ? val[i] : overutil_thres[i]; } overutil_thresh_chg_notify(); return count; } static ssize_t show_overutil(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int len = 0, i; unsigned int max_len = 4096; for (i = 0; i < OVERUTIL_THRESHOLD_SIZE; i++) { len += snprintf(buf+len, max_len-len, "over_util threshold[%u]=%d max=100\n", i, overutil_thres[i]); } #ifdef CONFIG_MTK_SCHED_RQAVG_KS len += get_overutil_stats(buf+len, max_len-len); #endif return len; } #else /* CONFIG_MTK_CORE_CTL is not set */ void set_overutil_threshold(int val) { overutil_threshold = (int)val; #ifdef CONFIG_MTK_SCHED_RQAVG_KS overutil_thresh_chg_notify(); #endif } EXPORT_SYMBOL(set_overutil_threshold); /* For read/write utilization related settings */ static ssize_t store_overutil(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; if (sscanf(buf, "%iu", &val) != 0) { if (val >= 0 && val <= 100) set_overutil_threshold(val); } return count; } static ssize_t show_overutil(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int len = 0; unsigned int max_len = 4096; len += snprintf(buf, max_len, "overutilization threshold=%d max=100\n\n", overutil_threshold); #ifdef CONFIG_MTK_SCHED_RQAVG_KS len += get_overutil_stats(buf+len, max_len-len); #endif return len; } #endif /* end of CONFIG_MTK_CORE_CTL */ static struct kobj_attribute over_util_attr = __ATTR(over_util, 0600 /* S_IWUSR | S_IRUSR*/, show_overutil, store_overutil); /* For read/write threshold for average heavy task */ static ssize_t store_avg_heavy_task_thresh(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; if (sscanf(buf, "%iu", &val) != 0) { if (val >= 0 && val < 1024) { heavy_task_threshold2 = val; #ifdef CONFIG_MTK_SCHED_RQAVG_KS heavy_thresh_chg_notify(); #endif } } return count; } static ssize_t show_avg_heavy_task_thresh(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int len = 0; unsigned int max_len = 4096; len += snprintf(buf, max_len, "%d\n", heavy_task_threshold2); return len; } static struct kobj_attribute avg_htasks_thresh_attr = __ATTR(avg_htasks_thresh, 0600 /* S_IWUSR | S_IRUSR */, show_avg_heavy_task_thresh, store_avg_heavy_task_thresh); /* big task */ static ssize_t show_big_task(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int max_len = 4096; return show_btask(buf, max_len); } static struct kobj_attribute big_task_attr = __ATTR(big_task, 0400 /* S_IRUSR */, show_big_task, NULL); static struct attribute *rq_attrs[] = { &cpu_normalized_load_attr.attr, &def_timer_ms_attr.attr, &run_queue_avg_attr.attr, &run_queue_poll_ms_attr.attr, &hotplug_disabled_attr.attr, &htasks_attr.attr, &htask_cpucap_ctrl_attr.attr, &htasks_thresh_attr.attr, &avg_htasks_thresh_attr.attr, &avg_htasks_ac_attr.attr, &over_util_attr.attr, &big_task_attr.attr, NULL, }; static struct attribute_group rq_attr_group = { .attrs = rq_attrs, }; static int init_rq_attribs(void) { int err; rq_info.rq_avg = 0; rq_info.attr_group = &rq_attr_group; /* Create /sys/devices/system/cpu/rq-stats/... */ rq_info.kobj = kobject_create_and_add("rq-stats", &cpu_subsys.dev_root->kobj); if (!rq_info.kobj) return -ENOMEM; err = sysfs_create_group(rq_info.kobj, rq_info.attr_group); if (err) kobject_put(rq_info.kobj); else kobject_uevent(rq_info.kobj, KOBJ_ADD); return err; } static int __init rq_stats_init(void) { int ret = 0; int i; #ifdef CONFIG_CPU_FREQ struct cpufreq_policy *cpu_policy; cpu_policy = kmalloc(sizeof(struct cpufreq_policy), GFP_KERNEL); if (!cpu_policy) { rq_info.init = 0; return -ENODATA; } #endif /* Bail out if this is not an SMP Target */ #ifndef CONFIG_SMP #ifdef CONFIG_CPU_FREQ kfree(cpu_policy); #endif rq_info.init = 0; return -ENODATA; #endif #ifdef CONFIG_MTK_SCHED_RQAVG_US_ENABLE_WQ rq_wq = create_singlethread_workqueue("rq_stats"); if (!rq_wq) return -ENODATA; INIT_WORK(&rq_info.def_timer_work, def_work_fn); #endif spin_lock_init(&rqavg_lock); rq_info.rq_poll_jiffies = DEFAULT_RQ_POLL_JIFFIES; rq_info.def_timer_jiffies = DEFAULT_DEF_TIMER_JIFFIES; rq_info.rq_poll_last_jiffy = 0; rq_info.def_timer_last_jiffy = 0; rq_info.hotplug_disabled = 0; ret = init_rq_attribs(); for_each_possible_cpu(i) { struct cpu_load_data *pcpu = &per_cpu(cpuload, i); spin_lock_init(&pcpu->cpu_load_lock); pcpu->cur_freq = pcpu->policy_max = 1; #ifdef CONFIG_CPU_FREQ if (!cpufreq_get_policy(cpu_policy, i)) { pcpu->policy_max = cpu_policy->cpuinfo.max_freq; if (cpu_online(i)) pcpu->cur_freq = cpufreq_quick_get(i); cpumask_copy(pcpu->related_cpus, cpu_policy->cpus); } #endif /* CONFIG_CPU_FREQ */ if (!cpufreq_variant) pcpu->cur_freq = pcpu->policy_max; } #ifdef CONFIG_CPU_FREQ freq_transition.notifier_call = cpufreq_transition_handler; cpufreq_register_notifier(&freq_transition, CPUFREQ_TRANSITION_NOTIFIER); #endif /* CONFIG_CPU_FREQ */ rq_info.init = 1; #ifdef CONFIG_CPU_FREQ kfree(cpu_policy); #endif return ret; } late_initcall(rq_stats_init); static int __init rq_stats_early_init(void) { /* Bail out if this is not an SMP Target */ #ifndef CONFIG_SMP rq_info.init = 0; return -ENODATA; #endif pm_notifier(system_suspend_handler, 0); return 0; } core_initcall(rq_stats_early_init);