From: jbeulich@novell.com Subject: fold per-CPU accounting data into a structure Patch-mainline: n/a ... to simplify generated code, especially in timer_interrupt(). This becomes more important with more such data elements added (i.e. by patches.xen/xen-x86-xtime-lock). --- sle11sp1-2010-02-17.orig/arch/x86/kernel/time-xen.c 2010-02-18 17:30:48.000000000 +0100 +++ sle11sp1-2010-02-17/arch/x86/kernel/time-xen.c 2010-02-18 17:32:00.000000000 +0100 @@ -57,12 +57,15 @@ static u32 shadow_tv_version; /* Keep track of last time we did processing/updating of jiffies and xtime. */ static u64 processed_system_time; /* System time (ns) at last processing. */ -static DEFINE_PER_CPU(u64, processed_system_time); -static DEFINE_PER_CPU(u64, accounted_system_time); -/* How much CPU time was spent blocked and how much was 'stolen'? */ -static DEFINE_PER_CPU(u64, processed_stolen_time); -static DEFINE_PER_CPU(u64, processed_blocked_time); +struct local_time_info { + u64 processed_system; + u64 accounted_system; + /* How much CPU time was spent blocked and how much was 'stolen'? */ + u64 accounted_stolen; + u64 accounted_blocked; +}; +static DEFINE_PER_CPU(struct local_time_info, local_time); /* Current runstate of each CPU (updated automatically by the hypervisor). */ DEFINE_PER_CPU(struct vcpu_runstate_info, runstate); @@ -440,6 +443,7 @@ static irqreturn_t timer_interrupt(int i s64 delta, delta_cpu, stolen, blocked; unsigned int i, cpu = smp_processor_id(); struct shadow_time_info *shadow = &per_cpu(shadow_time, cpu); + struct local_time_info *local = &per_cpu(local_time, cpu); bool duty = false; struct vcpu_runstate_info runstate; @@ -468,7 +472,7 @@ static irqreturn_t timer_interrupt(int i delta = delta_cpu = shadow->system_timestamp + get_nsec_offset(shadow); delta -= processed_system_time; - delta_cpu -= per_cpu(processed_system_time, cpu); + delta_cpu -= local->processed_system; get_runstate_snapshot(&runstate); } while (!time_values_up_to_date()); @@ -482,10 +486,10 @@ static irqreturn_t timer_interrupt(int i "processed=%Lx/%Lx\n", cpu, delta, delta_cpu, shadow->system_timestamp, get_nsec_offset(shadow), blocked, - per_cpu(processed_system_time, cpu)); + local->processed_system); for_each_cpu_and(i, cpu_online_mask, cpumask_of(cpu)) printk(" %u: %Lx\n", i, - per_cpu(processed_system_time, i)); + per_cpu(local_time.processed_system, i)); } } else if (unlikely(delta_cpu < -(s64)permitted_clock_jitter)) { blocked = processed_system_time; @@ -496,10 +500,10 @@ static irqreturn_t timer_interrupt(int i " shadow=%Lx off=%Lx processed=%Lx/%Lx\n", cpu, delta_cpu, shadow->system_timestamp, get_nsec_offset(shadow), blocked, - per_cpu(processed_system_time, cpu)); + local->processed_system); for_each_cpu_and(i, cpu_online_mask, cpumask_of(cpu)) printk(" %u: %Lx\n", i, - per_cpu(processed_system_time, i)); + per_cpu(local_time.processed_system, i)); } } else if (duty) { /* System-wide jiffy work. */ @@ -524,11 +528,10 @@ static irqreturn_t timer_interrupt(int i } delta = delta_cpu; - delta_cpu += per_cpu(processed_system_time, cpu) - - per_cpu(accounted_system_time, cpu); + delta_cpu += local->processed_system - local->accounted_system; if (delta >= NS_PER_TICK) { do_div(delta, NS_PER_TICK); - per_cpu(processed_system_time, cpu) += delta * NS_PER_TICK; + local->processed_system += delta * NS_PER_TICK; } /* @@ -537,14 +540,14 @@ static irqreturn_t timer_interrupt(int i */ stolen = runstate.time[RUNSTATE_runnable] + runstate.time[RUNSTATE_offline] - - per_cpu(processed_stolen_time, cpu); + - local->accounted_stolen; if ((stolen > 0) && (delta_cpu > 0)) { delta_cpu -= stolen; if (unlikely(delta_cpu < 0)) stolen += delta_cpu; /* clamp local-time progress */ do_div(stolen, NS_PER_TICK); - per_cpu(processed_stolen_time, cpu) += stolen * NS_PER_TICK; - per_cpu(accounted_system_time, cpu) += stolen * NS_PER_TICK; + local->accounted_stolen += stolen * NS_PER_TICK; + local->accounted_system += stolen * NS_PER_TICK; account_steal_ticks(stolen); } @@ -553,21 +556,21 @@ static irqreturn_t timer_interrupt(int i * ensures that the ticks are accounted as idle/wait. */ blocked = runstate.time[RUNSTATE_blocked] - - per_cpu(processed_blocked_time, cpu); + - local->accounted_blocked; if ((blocked > 0) && (delta_cpu > 0)) { delta_cpu -= blocked; if (unlikely(delta_cpu < 0)) blocked += delta_cpu; /* clamp local-time progress */ do_div(blocked, NS_PER_TICK); - per_cpu(processed_blocked_time, cpu) += blocked * NS_PER_TICK; - per_cpu(accounted_system_time, cpu) += blocked * NS_PER_TICK; + local->accounted_blocked += blocked * NS_PER_TICK; + local->accounted_system += blocked * NS_PER_TICK; account_idle_ticks(blocked); } /* Account user/system ticks. */ if (delta_cpu > 0) { do_div(delta_cpu, NS_PER_TICK); - per_cpu(accounted_system_time, cpu) += delta_cpu * NS_PER_TICK; + local->accounted_system += delta_cpu * NS_PER_TICK; if (user_mode_vm(get_irq_regs())) account_user_time(current, (cputime_t)delta_cpu, (cputime_t)delta_cpu); @@ -606,9 +609,9 @@ static void init_missing_ticks_accountin { struct vcpu_runstate_info *runstate = setup_runstate_area(cpu); - per_cpu(processed_blocked_time, cpu) = + per_cpu(local_time.accounted_blocked, cpu) = runstate->time[RUNSTATE_blocked]; - per_cpu(processed_stolen_time, cpu) = + per_cpu(local_time.accounted_stolen, cpu) = runstate->time[RUNSTATE_runnable] + runstate->time[RUNSTATE_offline]; } @@ -668,8 +671,8 @@ static void xen_clocksource_resume(void) BUG(); } get_time_values_from_xen(cpu); - per_cpu(accounted_system_time, cpu) = - per_cpu(processed_system_time, cpu) = + per_cpu(local_time.accounted_system, cpu) = + per_cpu(local_time.processed_system, cpu) = per_cpu(shadow_time, 0).system_timestamp; init_missing_ticks_accounting(cpu); } @@ -770,8 +773,8 @@ void __init time_init(void) get_time_values_from_xen(0); processed_system_time = per_cpu(shadow_time, 0).system_timestamp; - per_cpu(processed_system_time, 0) = processed_system_time; - per_cpu(accounted_system_time, 0) = processed_system_time; + per_cpu(local_time.processed_system, 0) = processed_system_time; + per_cpu(local_time.accounted_system, 0) = processed_system_time; init_missing_ticks_accounting(0); clocksource_register(&clocksource_xen); @@ -849,7 +852,7 @@ static void stop_hz_timer(void) singleshot.timeout_abs_ns = jiffies_to_st(j); if (!singleshot.timeout_abs_ns) return; - local = per_cpu(processed_system_time, cpu); + local = per_cpu(local_time.processed_system, cpu); if ((s64)(singleshot.timeout_abs_ns - local) <= NS_PER_TICK) { cpumask_clear_cpu(cpu, nohz_cpu_mask); singleshot.timeout_abs_ns = local + NS_PER_TICK; @@ -918,8 +921,8 @@ int __cpuinit local_setup_timer(unsigned do { seq = read_seqbegin(&xtime_lock); /* Use cpu0 timestamp: cpu's shadow is not initialised yet. */ - per_cpu(accounted_system_time, cpu) = - per_cpu(processed_system_time, cpu) = + per_cpu(local_time.accounted_system, cpu) = + per_cpu(local_time.processed_system, cpu) = per_cpu(shadow_time, 0).system_timestamp; init_missing_ticks_accounting(cpu); } while (read_seqretry(&xtime_lock, seq));