]> git.openfabrics.org - ~shefty/rdma-dev.git/blob - arch/s390/oprofile/hwsampler.c
Merge branch 'for-tip' of git://git.kernel.org/pub/scm/linux/kernel/git/rric/oprofile...
[~shefty/rdma-dev.git] / arch / s390 / oprofile / hwsampler.c
1 /**
2  * arch/s390/oprofile/hwsampler.c
3  *
4  * Copyright IBM Corp. 2010
5  * Author: Heinz Graalfs <graalfs@de.ibm.com>
6  */
7
8 #include <linux/kernel_stat.h>
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/smp.h>
12 #include <linux/errno.h>
13 #include <linux/workqueue.h>
14 #include <linux/interrupt.h>
15 #include <linux/notifier.h>
16 #include <linux/cpu.h>
17 #include <linux/semaphore.h>
18 #include <linux/oom.h>
19 #include <linux/oprofile.h>
20
21 #include <asm/lowcore.h>
22 #include <asm/irq.h>
23
24 #include "hwsampler.h"
25 #include "op_counter.h"
26
27 #define MAX_NUM_SDB 511
28 #define MIN_NUM_SDB 1
29
30 #define ALERT_REQ_MASK   0x4000000000000000ul
31 #define BUFFER_FULL_MASK 0x8000000000000000ul
32
33 #define EI_IEA      (1 << 31)   /* invalid entry address              */
34 #define EI_ISE      (1 << 30)   /* incorrect SDBT entry               */
35 #define EI_PRA      (1 << 29)   /* program request alert              */
36 #define EI_SACA     (1 << 23)   /* sampler authorization change alert */
37 #define EI_LSDA     (1 << 22)   /* loss of sample data alert          */
38
39 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
40
41 struct hws_execute_parms {
42         void *buffer;
43         signed int rc;
44 };
45
46 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
47 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
48
49 static DEFINE_MUTEX(hws_sem);
50 static DEFINE_MUTEX(hws_sem_oom);
51
52 static unsigned char hws_flush_all;
53 static unsigned int hws_oom;
54 static struct workqueue_struct *hws_wq;
55
56 static unsigned int hws_state;
57 enum {
58         HWS_INIT = 1,
59         HWS_DEALLOCATED,
60         HWS_STOPPED,
61         HWS_STARTED,
62         HWS_STOPPING };
63
64 /* set to 1 if called by kernel during memory allocation */
65 static unsigned char oom_killer_was_active;
66 /* size of SDBT and SDB as of allocate API */
67 static unsigned long num_sdbt = 100;
68 static unsigned long num_sdb = 511;
69 /* sampling interval (machine cycles) */
70 static unsigned long interval;
71
72 static unsigned long min_sampler_rate;
73 static unsigned long max_sampler_rate;
74
75 static int ssctl(void *buffer)
76 {
77         int cc;
78
79         /* set in order to detect a program check */
80         cc = 1;
81
82         asm volatile(
83                 "0: .insn s,0xB2870000,0(%1)\n"
84                 "1: ipm %0\n"
85                 "   srl %0,28\n"
86                 "2:\n"
87                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
88                 : "+d" (cc), "+a" (buffer)
89                 : "m" (*((struct hws_ssctl_request_block *)buffer))
90                 : "cc", "memory");
91
92         return cc ? -EINVAL : 0 ;
93 }
94
95 static int qsi(void *buffer)
96 {
97         int cc;
98         cc = 1;
99
100         asm volatile(
101                 "0: .insn s,0xB2860000,0(%1)\n"
102                 "1: lhi %0,0\n"
103                 "2:\n"
104                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
105                 : "=d" (cc), "+a" (buffer)
106                 : "m" (*((struct hws_qsi_info_block *)buffer))
107                 : "cc", "memory");
108
109         return cc ? -EINVAL : 0;
110 }
111
112 static void execute_qsi(void *parms)
113 {
114         struct hws_execute_parms *ep = parms;
115
116         ep->rc = qsi(ep->buffer);
117 }
118
119 static void execute_ssctl(void *parms)
120 {
121         struct hws_execute_parms *ep = parms;
122
123         ep->rc = ssctl(ep->buffer);
124 }
125
126 static int smp_ctl_ssctl_stop(int cpu)
127 {
128         int rc;
129         struct hws_execute_parms ep;
130         struct hws_cpu_buffer *cb;
131
132         cb = &per_cpu(sampler_cpu_buffer, cpu);
133
134         cb->ssctl.es = 0;
135         cb->ssctl.cs = 0;
136
137         ep.buffer = &cb->ssctl;
138         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
139         rc = ep.rc;
140         if (rc) {
141                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
142                 dump_stack();
143         }
144
145         ep.buffer = &cb->qsi;
146         smp_call_function_single(cpu, execute_qsi, &ep, 1);
147
148         if (cb->qsi.es || cb->qsi.cs) {
149                 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
150                 dump_stack();
151         }
152
153         return rc;
154 }
155
156 static int smp_ctl_ssctl_deactivate(int cpu)
157 {
158         int rc;
159         struct hws_execute_parms ep;
160         struct hws_cpu_buffer *cb;
161
162         cb = &per_cpu(sampler_cpu_buffer, cpu);
163
164         cb->ssctl.es = 1;
165         cb->ssctl.cs = 0;
166
167         ep.buffer = &cb->ssctl;
168         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
169         rc = ep.rc;
170         if (rc)
171                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
172
173         ep.buffer = &cb->qsi;
174         smp_call_function_single(cpu, execute_qsi, &ep, 1);
175
176         if (cb->qsi.cs)
177                 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
178
179         return rc;
180 }
181
182 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
183 {
184         int rc;
185         struct hws_execute_parms ep;
186         struct hws_cpu_buffer *cb;
187
188         cb = &per_cpu(sampler_cpu_buffer, cpu);
189
190         cb->ssctl.h = 1;
191         cb->ssctl.tear = cb->first_sdbt;
192         cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
193         cb->ssctl.interval = interval;
194         cb->ssctl.es = 1;
195         cb->ssctl.cs = 1;
196
197         ep.buffer = &cb->ssctl;
198         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
199         rc = ep.rc;
200         if (rc)
201                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
202
203         ep.buffer = &cb->qsi;
204         smp_call_function_single(cpu, execute_qsi, &ep, 1);
205         if (ep.rc)
206                 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
207
208         return rc;
209 }
210
211 static int smp_ctl_qsi(int cpu)
212 {
213         struct hws_execute_parms ep;
214         struct hws_cpu_buffer *cb;
215
216         cb = &per_cpu(sampler_cpu_buffer, cpu);
217
218         ep.buffer = &cb->qsi;
219         smp_call_function_single(cpu, execute_qsi, &ep, 1);
220
221         return ep.rc;
222 }
223
224 static inline unsigned long *trailer_entry_ptr(unsigned long v)
225 {
226         void *ret;
227
228         ret = (void *)v;
229         ret += PAGE_SIZE;
230         ret -= sizeof(struct hws_trailer_entry);
231
232         return (unsigned long *) ret;
233 }
234
235 /* prototypes for external interrupt handler and worker */
236 static void hws_ext_handler(unsigned int ext_int_code,
237                                 unsigned int param32, unsigned long param64);
238
239 static void worker(struct work_struct *work);
240
241 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
242                                 unsigned long *dear);
243
244 static void init_all_cpu_buffers(void)
245 {
246         int cpu;
247         struct hws_cpu_buffer *cb;
248
249         for_each_online_cpu(cpu) {
250                 cb = &per_cpu(sampler_cpu_buffer, cpu);
251                 memset(cb, 0, sizeof(struct hws_cpu_buffer));
252         }
253 }
254
255 static int is_link_entry(unsigned long *s)
256 {
257         return *s & 0x1ul ? 1 : 0;
258 }
259
260 static unsigned long *get_next_sdbt(unsigned long *s)
261 {
262         return (unsigned long *) (*s & ~0x1ul);
263 }
264
265 static int prepare_cpu_buffers(void)
266 {
267         int cpu;
268         int rc;
269         struct hws_cpu_buffer *cb;
270
271         rc = 0;
272         for_each_online_cpu(cpu) {
273                 cb = &per_cpu(sampler_cpu_buffer, cpu);
274                 atomic_set(&cb->ext_params, 0);
275                 cb->worker_entry = 0;
276                 cb->sample_overflow = 0;
277                 cb->req_alert = 0;
278                 cb->incorrect_sdbt_entry = 0;
279                 cb->invalid_entry_address = 0;
280                 cb->loss_of_sample_data = 0;
281                 cb->sample_auth_change_alert = 0;
282                 cb->finish = 0;
283                 cb->oom = 0;
284                 cb->stop_mode = 0;
285         }
286
287         return rc;
288 }
289
290 /*
291  * allocate_sdbt() - allocate sampler memory
292  * @cpu: the cpu for which sampler memory is allocated
293  *
294  * A 4K page is allocated for each requested SDBT.
295  * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
296  * Set ALERT_REQ mask in each SDBs trailer.
297  * Returns zero if successful, <0 otherwise.
298  */
299 static int allocate_sdbt(int cpu)
300 {
301         int j, k, rc;
302         unsigned long *sdbt;
303         unsigned long  sdb;
304         unsigned long *tail;
305         unsigned long *trailer;
306         struct hws_cpu_buffer *cb;
307
308         cb = &per_cpu(sampler_cpu_buffer, cpu);
309
310         if (cb->first_sdbt)
311                 return -EINVAL;
312
313         sdbt = NULL;
314         tail = sdbt;
315
316         for (j = 0; j < num_sdbt; j++) {
317                 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
318
319                 mutex_lock(&hws_sem_oom);
320                 /* OOM killer might have been activated */
321                 barrier();
322                 if (oom_killer_was_active || !sdbt) {
323                         if (sdbt)
324                                 free_page((unsigned long)sdbt);
325
326                         goto allocate_sdbt_error;
327                 }
328                 if (cb->first_sdbt == 0)
329                         cb->first_sdbt = (unsigned long)sdbt;
330
331                 /* link current page to tail of chain */
332                 if (tail)
333                         *tail = (unsigned long)(void *)sdbt + 1;
334
335                 mutex_unlock(&hws_sem_oom);
336
337                 for (k = 0; k < num_sdb; k++) {
338                         /* get and set SDB page */
339                         sdb = get_zeroed_page(GFP_KERNEL);
340
341                         mutex_lock(&hws_sem_oom);
342                         /* OOM killer might have been activated */
343                         barrier();
344                         if (oom_killer_was_active || !sdb) {
345                                 if (sdb)
346                                         free_page(sdb);
347
348                                 goto allocate_sdbt_error;
349                         }
350                         *sdbt = sdb;
351                         trailer = trailer_entry_ptr(*sdbt);
352                         *trailer = ALERT_REQ_MASK;
353                         sdbt++;
354                         mutex_unlock(&hws_sem_oom);
355                 }
356                 tail = sdbt;
357         }
358         mutex_lock(&hws_sem_oom);
359         if (oom_killer_was_active)
360                 goto allocate_sdbt_error;
361
362         rc = 0;
363         if (tail)
364                 *tail = (unsigned long)
365                         ((void *)cb->first_sdbt) + 1;
366
367 allocate_sdbt_exit:
368         mutex_unlock(&hws_sem_oom);
369         return rc;
370
371 allocate_sdbt_error:
372         rc = -ENOMEM;
373         goto allocate_sdbt_exit;
374 }
375
376 /*
377  * deallocate_sdbt() - deallocate all sampler memory
378  *
379  * For each online CPU all SDBT trees are deallocated.
380  * Returns the number of freed pages.
381  */
382 static int deallocate_sdbt(void)
383 {
384         int cpu;
385         int counter;
386
387         counter = 0;
388
389         for_each_online_cpu(cpu) {
390                 unsigned long start;
391                 unsigned long sdbt;
392                 unsigned long *curr;
393                 struct hws_cpu_buffer *cb;
394
395                 cb = &per_cpu(sampler_cpu_buffer, cpu);
396
397                 if (!cb->first_sdbt)
398                         continue;
399
400                 sdbt = cb->first_sdbt;
401                 curr = (unsigned long *) sdbt;
402                 start = sdbt;
403
404                 /* we'll free the SDBT after all SDBs are processed... */
405                 while (1) {
406                         if (!*curr || !sdbt)
407                                 break;
408
409                         /* watch for link entry reset if found */
410                         if (is_link_entry(curr)) {
411                                 curr = get_next_sdbt(curr);
412                                 if (sdbt)
413                                         free_page(sdbt);
414
415                                 /* we are done if we reach the start */
416                                 if ((unsigned long) curr == start)
417                                         break;
418                                 else
419                                         sdbt = (unsigned long) curr;
420                         } else {
421                                 /* process SDB pointer */
422                                 if (*curr) {
423                                         free_page(*curr);
424                                         curr++;
425                                 }
426                         }
427                         counter++;
428                 }
429                 cb->first_sdbt = 0;
430         }
431         return counter;
432 }
433
434 static int start_sampling(int cpu)
435 {
436         int rc;
437         struct hws_cpu_buffer *cb;
438
439         cb = &per_cpu(sampler_cpu_buffer, cpu);
440         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
441         if (rc) {
442                 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
443                 goto start_exit;
444         }
445
446         rc = -EINVAL;
447         if (!cb->qsi.es) {
448                 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
449                 goto start_exit;
450         }
451
452         if (!cb->qsi.cs) {
453                 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
454                 goto start_exit;
455         }
456
457         printk(KERN_INFO
458                 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
459                 cpu, interval);
460
461         rc = 0;
462
463 start_exit:
464         return rc;
465 }
466
467 static int stop_sampling(int cpu)
468 {
469         unsigned long v;
470         int rc;
471         struct hws_cpu_buffer *cb;
472
473         rc = smp_ctl_qsi(cpu);
474         WARN_ON(rc);
475
476         cb = &per_cpu(sampler_cpu_buffer, cpu);
477         if (!rc && !cb->qsi.es)
478                 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
479
480         rc = smp_ctl_ssctl_stop(cpu);
481         if (rc) {
482                 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
483                                 cpu, rc);
484                 goto stop_exit;
485         }
486
487         printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
488
489 stop_exit:
490         v = cb->req_alert;
491         if (v)
492                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
493                                 " count=%lu.\n", cpu, v);
494
495         v = cb->loss_of_sample_data;
496         if (v)
497                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
498                                 " count=%lu.\n", cpu, v);
499
500         v = cb->invalid_entry_address;
501         if (v)
502                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
503                                 " count=%lu.\n", cpu, v);
504
505         v = cb->incorrect_sdbt_entry;
506         if (v)
507                 printk(KERN_ERR
508                                 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
509                                 " count=%lu.\n", cpu, v);
510
511         v = cb->sample_auth_change_alert;
512         if (v)
513                 printk(KERN_ERR
514                                 "hwsampler: CPU %d CPUMF Sample authorization change,"
515                                 " count=%lu.\n", cpu, v);
516
517         return rc;
518 }
519
520 static int check_hardware_prerequisites(void)
521 {
522         if (!test_facility(68))
523                 return -EOPNOTSUPP;
524         return 0;
525 }
526 /*
527  * hws_oom_callback() - the OOM callback function
528  *
529  * In case the callback is invoked during memory allocation for the
530  *  hw sampler, all obtained memory is deallocated and a flag is set
531  *  so main sampler memory allocation can exit with a failure code.
532  * In case the callback is invoked during sampling the hw sampler
533  *  is deactivated for all CPUs.
534  */
535 static int hws_oom_callback(struct notifier_block *nfb,
536         unsigned long dummy, void *parm)
537 {
538         unsigned long *freed;
539         int cpu;
540         struct hws_cpu_buffer *cb;
541
542         freed = parm;
543
544         mutex_lock(&hws_sem_oom);
545
546         if (hws_state == HWS_DEALLOCATED) {
547                 /* during memory allocation */
548                 if (oom_killer_was_active == 0) {
549                         oom_killer_was_active = 1;
550                         *freed += deallocate_sdbt();
551                 }
552         } else {
553                 int i;
554                 cpu = get_cpu();
555                 cb = &per_cpu(sampler_cpu_buffer, cpu);
556
557                 if (!cb->oom) {
558                         for_each_online_cpu(i) {
559                                 smp_ctl_ssctl_deactivate(i);
560                                 cb->oom = 1;
561                         }
562                         cb->finish = 1;
563
564                         printk(KERN_INFO
565                                 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
566                                 cpu);
567                 }
568         }
569
570         mutex_unlock(&hws_sem_oom);
571
572         return NOTIFY_OK;
573 }
574
575 static struct notifier_block hws_oom_notifier = {
576         .notifier_call = hws_oom_callback
577 };
578
579 static int hws_cpu_callback(struct notifier_block *nfb,
580         unsigned long action, void *hcpu)
581 {
582         /* We do not have sampler space available for all possible CPUs.
583            All CPUs should be online when hw sampling is activated. */
584         return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
585 }
586
587 static struct notifier_block hws_cpu_notifier = {
588         .notifier_call = hws_cpu_callback
589 };
590
591 /**
592  * hwsampler_deactivate() - set hardware sampling temporarily inactive
593  * @cpu:  specifies the CPU to be set inactive.
594  *
595  * Returns 0 on success, !0 on failure.
596  */
597 int hwsampler_deactivate(unsigned int cpu)
598 {
599         /*
600          * Deactivate hw sampling temporarily and flush the buffer
601          * by pushing all the pending samples to oprofile buffer.
602          *
603          * This function can be called under one of the following conditions:
604          *     Memory unmap, task is exiting.
605          */
606         int rc;
607         struct hws_cpu_buffer *cb;
608
609         rc = 0;
610         mutex_lock(&hws_sem);
611
612         cb = &per_cpu(sampler_cpu_buffer, cpu);
613         if (hws_state == HWS_STARTED) {
614                 rc = smp_ctl_qsi(cpu);
615                 WARN_ON(rc);
616                 if (cb->qsi.cs) {
617                         rc = smp_ctl_ssctl_deactivate(cpu);
618                         if (rc) {
619                                 printk(KERN_INFO
620                                 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
621                                 cb->finish = 1;
622                                 hws_state = HWS_STOPPING;
623                         } else  {
624                                 hws_flush_all = 1;
625                                 /* Add work to queue to read pending samples.*/
626                                 queue_work_on(cpu, hws_wq, &cb->worker);
627                         }
628                 }
629         }
630         mutex_unlock(&hws_sem);
631
632         if (hws_wq)
633                 flush_workqueue(hws_wq);
634
635         return rc;
636 }
637
638 /**
639  * hwsampler_activate() - activate/resume hardware sampling which was deactivated
640  * @cpu:  specifies the CPU to be set active.
641  *
642  * Returns 0 on success, !0 on failure.
643  */
644 int hwsampler_activate(unsigned int cpu)
645 {
646         /*
647          * Re-activate hw sampling. This should be called in pair with
648          * hwsampler_deactivate().
649          */
650         int rc;
651         struct hws_cpu_buffer *cb;
652
653         rc = 0;
654         mutex_lock(&hws_sem);
655
656         cb = &per_cpu(sampler_cpu_buffer, cpu);
657         if (hws_state == HWS_STARTED) {
658                 rc = smp_ctl_qsi(cpu);
659                 WARN_ON(rc);
660                 if (!cb->qsi.cs) {
661                         hws_flush_all = 0;
662                         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
663                         if (rc) {
664                                 printk(KERN_ERR
665                                 "CPU %d, CPUMF activate sampling failed.\n",
666                                          cpu);
667                         }
668                 }
669         }
670
671         mutex_unlock(&hws_sem);
672
673         return rc;
674 }
675
676 static void hws_ext_handler(unsigned int ext_int_code,
677                             unsigned int param32, unsigned long param64)
678 {
679         struct hws_cpu_buffer *cb;
680
681         kstat_cpu(smp_processor_id()).irqs[EXTINT_CPM]++;
682         cb = &__get_cpu_var(sampler_cpu_buffer);
683         atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
684         if (hws_wq)
685                 queue_work(hws_wq, &cb->worker);
686 }
687
688 static int check_qsi_on_setup(void)
689 {
690         int rc;
691         unsigned int cpu;
692         struct hws_cpu_buffer *cb;
693
694         for_each_online_cpu(cpu) {
695                 cb = &per_cpu(sampler_cpu_buffer, cpu);
696                 rc = smp_ctl_qsi(cpu);
697                 WARN_ON(rc);
698                 if (rc)
699                         return -EOPNOTSUPP;
700
701                 if (!cb->qsi.as) {
702                         printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
703                         return -EINVAL;
704                 }
705
706                 if (cb->qsi.es) {
707                         printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
708                         rc = smp_ctl_ssctl_stop(cpu);
709                         if (rc)
710                                 return -EINVAL;
711
712                         printk(KERN_INFO
713                                 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
714                 }
715         }
716         return 0;
717 }
718
719 static int check_qsi_on_start(void)
720 {
721         unsigned int cpu;
722         int rc;
723         struct hws_cpu_buffer *cb;
724
725         for_each_online_cpu(cpu) {
726                 cb = &per_cpu(sampler_cpu_buffer, cpu);
727                 rc = smp_ctl_qsi(cpu);
728                 WARN_ON(rc);
729
730                 if (!cb->qsi.as)
731                         return -EINVAL;
732
733                 if (cb->qsi.es)
734                         return -EINVAL;
735
736                 if (cb->qsi.cs)
737                         return -EINVAL;
738         }
739         return 0;
740 }
741
742 static void worker_on_start(unsigned int cpu)
743 {
744         struct hws_cpu_buffer *cb;
745
746         cb = &per_cpu(sampler_cpu_buffer, cpu);
747         cb->worker_entry = cb->first_sdbt;
748 }
749
750 static int worker_check_error(unsigned int cpu, int ext_params)
751 {
752         int rc;
753         unsigned long *sdbt;
754         struct hws_cpu_buffer *cb;
755
756         rc = 0;
757         cb = &per_cpu(sampler_cpu_buffer, cpu);
758         sdbt = (unsigned long *) cb->worker_entry;
759
760         if (!sdbt || !*sdbt)
761                 return -EINVAL;
762
763         if (ext_params & EI_PRA)
764                 cb->req_alert++;
765
766         if (ext_params & EI_LSDA)
767                 cb->loss_of_sample_data++;
768
769         if (ext_params & EI_IEA) {
770                 cb->invalid_entry_address++;
771                 rc = -EINVAL;
772         }
773
774         if (ext_params & EI_ISE) {
775                 cb->incorrect_sdbt_entry++;
776                 rc = -EINVAL;
777         }
778
779         if (ext_params & EI_SACA) {
780                 cb->sample_auth_change_alert++;
781                 rc = -EINVAL;
782         }
783
784         return rc;
785 }
786
787 static void worker_on_finish(unsigned int cpu)
788 {
789         int rc, i;
790         struct hws_cpu_buffer *cb;
791
792         cb = &per_cpu(sampler_cpu_buffer, cpu);
793
794         if (cb->finish) {
795                 rc = smp_ctl_qsi(cpu);
796                 WARN_ON(rc);
797                 if (cb->qsi.es) {
798                         printk(KERN_INFO
799                                 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
800                                 cpu);
801                         rc = smp_ctl_ssctl_stop(cpu);
802                         if (rc)
803                                 printk(KERN_INFO
804                                         "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
805                                         cpu);
806
807                         for_each_online_cpu(i) {
808                                 if (i == cpu)
809                                         continue;
810                                 if (!cb->finish) {
811                                         cb->finish = 1;
812                                         queue_work_on(i, hws_wq,
813                                                 &cb->worker);
814                                 }
815                         }
816                 }
817         }
818 }
819
820 static void worker_on_interrupt(unsigned int cpu)
821 {
822         unsigned long *sdbt;
823         unsigned char done;
824         struct hws_cpu_buffer *cb;
825
826         cb = &per_cpu(sampler_cpu_buffer, cpu);
827
828         sdbt = (unsigned long *) cb->worker_entry;
829
830         done = 0;
831         /* do not proceed if stop was entered,
832          * forget the buffers not yet processed */
833         while (!done && !cb->stop_mode) {
834                 unsigned long *trailer;
835                 struct hws_trailer_entry *te;
836                 unsigned long *dear = 0;
837
838                 trailer = trailer_entry_ptr(*sdbt);
839                 /* leave loop if no more work to do */
840                 if (!(*trailer & BUFFER_FULL_MASK)) {
841                         done = 1;
842                         if (!hws_flush_all)
843                                 continue;
844                 }
845
846                 te = (struct hws_trailer_entry *)trailer;
847                 cb->sample_overflow += te->overflow;
848
849                 add_samples_to_oprofile(cpu, sdbt, dear);
850
851                 /* reset trailer */
852                 xchg((unsigned char *) te, 0x40);
853
854                 /* advance to next sdb slot in current sdbt */
855                 sdbt++;
856                 /* in case link bit is set use address w/o link bit */
857                 if (is_link_entry(sdbt))
858                         sdbt = get_next_sdbt(sdbt);
859
860                 cb->worker_entry = (unsigned long)sdbt;
861         }
862 }
863
864 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
865                 unsigned long *dear)
866 {
867         struct hws_data_entry *sample_data_ptr;
868         unsigned long *trailer;
869
870         trailer = trailer_entry_ptr(*sdbt);
871         if (dear) {
872                 if (dear > trailer)
873                         return;
874                 trailer = dear;
875         }
876
877         sample_data_ptr = (struct hws_data_entry *)(*sdbt);
878
879         while ((unsigned long *)sample_data_ptr < trailer) {
880                 struct pt_regs *regs = NULL;
881                 struct task_struct *tsk = NULL;
882
883                 /*
884                  * Check sampling mode, 1 indicates basic (=customer) sampling
885                  * mode.
886                  */
887                 if (sample_data_ptr->def != 1) {
888                         /* sample slot is not yet written */
889                         break;
890                 } else {
891                         /* make sure we don't use it twice,
892                          * the next time the sampler will set it again */
893                         sample_data_ptr->def = 0;
894                 }
895
896                 /* Get pt_regs. */
897                 if (sample_data_ptr->P == 1) {
898                         /* userspace sample */
899                         unsigned int pid = sample_data_ptr->prim_asn;
900                         if (!counter_config.user)
901                                 goto skip_sample;
902                         rcu_read_lock();
903                         tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
904                         if (tsk)
905                                 regs = task_pt_regs(tsk);
906                         rcu_read_unlock();
907                 } else {
908                         /* kernelspace sample */
909                         if (!counter_config.kernel)
910                                 goto skip_sample;
911                         regs = task_pt_regs(current);
912                 }
913
914                 mutex_lock(&hws_sem);
915                 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
916                                 !sample_data_ptr->P, tsk);
917                 mutex_unlock(&hws_sem);
918         skip_sample:
919                 sample_data_ptr++;
920         }
921 }
922
923 static void worker(struct work_struct *work)
924 {
925         unsigned int cpu;
926         int ext_params;
927         struct hws_cpu_buffer *cb;
928
929         cb = container_of(work, struct hws_cpu_buffer, worker);
930         cpu = smp_processor_id();
931         ext_params = atomic_xchg(&cb->ext_params, 0);
932
933         if (!cb->worker_entry)
934                 worker_on_start(cpu);
935
936         if (worker_check_error(cpu, ext_params))
937                 return;
938
939         if (!cb->finish)
940                 worker_on_interrupt(cpu);
941
942         if (cb->finish)
943                 worker_on_finish(cpu);
944 }
945
946 /**
947  * hwsampler_allocate() - allocate memory for the hardware sampler
948  * @sdbt:  number of SDBTs per online CPU (must be > 0)
949  * @sdb:   number of SDBs per SDBT (minimum 1, maximum 511)
950  *
951  * Returns 0 on success, !0 on failure.
952  */
953 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
954 {
955         int cpu, rc;
956         mutex_lock(&hws_sem);
957
958         rc = -EINVAL;
959         if (hws_state != HWS_DEALLOCATED)
960                 goto allocate_exit;
961
962         if (sdbt < 1)
963                 goto allocate_exit;
964
965         if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
966                 goto allocate_exit;
967
968         num_sdbt = sdbt;
969         num_sdb = sdb;
970
971         oom_killer_was_active = 0;
972         register_oom_notifier(&hws_oom_notifier);
973
974         for_each_online_cpu(cpu) {
975                 if (allocate_sdbt(cpu)) {
976                         unregister_oom_notifier(&hws_oom_notifier);
977                         goto allocate_error;
978                 }
979         }
980         unregister_oom_notifier(&hws_oom_notifier);
981         if (oom_killer_was_active)
982                 goto allocate_error;
983
984         hws_state = HWS_STOPPED;
985         rc = 0;
986
987 allocate_exit:
988         mutex_unlock(&hws_sem);
989         return rc;
990
991 allocate_error:
992         rc = -ENOMEM;
993         printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
994         goto allocate_exit;
995 }
996
997 /**
998  * hwsampler_deallocate() - deallocate hardware sampler memory
999  *
1000  * Returns 0 on success, !0 on failure.
1001  */
1002 int hwsampler_deallocate(void)
1003 {
1004         int rc;
1005
1006         mutex_lock(&hws_sem);
1007
1008         rc = -EINVAL;
1009         if (hws_state != HWS_STOPPED)
1010                 goto deallocate_exit;
1011
1012         ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1013         deallocate_sdbt();
1014
1015         hws_state = HWS_DEALLOCATED;
1016         rc = 0;
1017
1018 deallocate_exit:
1019         mutex_unlock(&hws_sem);
1020
1021         return rc;
1022 }
1023
1024 unsigned long hwsampler_query_min_interval(void)
1025 {
1026         return min_sampler_rate;
1027 }
1028
1029 unsigned long hwsampler_query_max_interval(void)
1030 {
1031         return max_sampler_rate;
1032 }
1033
1034 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1035 {
1036         struct hws_cpu_buffer *cb;
1037
1038         cb = &per_cpu(sampler_cpu_buffer, cpu);
1039
1040         return cb->sample_overflow;
1041 }
1042
1043 int hwsampler_setup(void)
1044 {
1045         int rc;
1046         int cpu;
1047         struct hws_cpu_buffer *cb;
1048
1049         mutex_lock(&hws_sem);
1050
1051         rc = -EINVAL;
1052         if (hws_state)
1053                 goto setup_exit;
1054
1055         hws_state = HWS_INIT;
1056
1057         init_all_cpu_buffers();
1058
1059         rc = check_hardware_prerequisites();
1060         if (rc)
1061                 goto setup_exit;
1062
1063         rc = check_qsi_on_setup();
1064         if (rc)
1065                 goto setup_exit;
1066
1067         rc = -EINVAL;
1068         hws_wq = create_workqueue("hwsampler");
1069         if (!hws_wq)
1070                 goto setup_exit;
1071
1072         register_cpu_notifier(&hws_cpu_notifier);
1073
1074         for_each_online_cpu(cpu) {
1075                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1076                 INIT_WORK(&cb->worker, worker);
1077                 rc = smp_ctl_qsi(cpu);
1078                 WARN_ON(rc);
1079                 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1080                         if (min_sampler_rate) {
1081                                 printk(KERN_WARNING
1082                                         "hwsampler: different min sampler rate values.\n");
1083                                 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1084                                         min_sampler_rate =
1085                                                 cb->qsi.min_sampl_rate;
1086                         } else
1087                                 min_sampler_rate = cb->qsi.min_sampl_rate;
1088                 }
1089                 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1090                         if (max_sampler_rate) {
1091                                 printk(KERN_WARNING
1092                                         "hwsampler: different max sampler rate values.\n");
1093                                 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1094                                         max_sampler_rate =
1095                                                 cb->qsi.max_sampl_rate;
1096                         } else
1097                                 max_sampler_rate = cb->qsi.max_sampl_rate;
1098                 }
1099         }
1100         register_external_interrupt(0x1407, hws_ext_handler);
1101
1102         hws_state = HWS_DEALLOCATED;
1103         rc = 0;
1104
1105 setup_exit:
1106         mutex_unlock(&hws_sem);
1107         return rc;
1108 }
1109
1110 int hwsampler_shutdown(void)
1111 {
1112         int rc;
1113
1114         mutex_lock(&hws_sem);
1115
1116         rc = -EINVAL;
1117         if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1118                 mutex_unlock(&hws_sem);
1119
1120                 if (hws_wq)
1121                         flush_workqueue(hws_wq);
1122
1123                 mutex_lock(&hws_sem);
1124
1125                 if (hws_state == HWS_STOPPED) {
1126                         ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1127                         deallocate_sdbt();
1128                 }
1129                 if (hws_wq) {
1130                         destroy_workqueue(hws_wq);
1131                         hws_wq = NULL;
1132                 }
1133
1134                 unregister_external_interrupt(0x1407, hws_ext_handler);
1135                 hws_state = HWS_INIT;
1136                 rc = 0;
1137         }
1138         mutex_unlock(&hws_sem);
1139
1140         unregister_cpu_notifier(&hws_cpu_notifier);
1141
1142         return rc;
1143 }
1144
1145 /**
1146  * hwsampler_start_all() - start hardware sampling on all online CPUs
1147  * @rate:  specifies the used interval when samples are taken
1148  *
1149  * Returns 0 on success, !0 on failure.
1150  */
1151 int hwsampler_start_all(unsigned long rate)
1152 {
1153         int rc, cpu;
1154
1155         mutex_lock(&hws_sem);
1156
1157         hws_oom = 0;
1158
1159         rc = -EINVAL;
1160         if (hws_state != HWS_STOPPED)
1161                 goto start_all_exit;
1162
1163         interval = rate;
1164
1165         /* fail if rate is not valid */
1166         if (interval < min_sampler_rate || interval > max_sampler_rate)
1167                 goto start_all_exit;
1168
1169         rc = check_qsi_on_start();
1170         if (rc)
1171                 goto start_all_exit;
1172
1173         rc = prepare_cpu_buffers();
1174         if (rc)
1175                 goto start_all_exit;
1176
1177         for_each_online_cpu(cpu) {
1178                 rc = start_sampling(cpu);
1179                 if (rc)
1180                         break;
1181         }
1182         if (rc) {
1183                 for_each_online_cpu(cpu) {
1184                         stop_sampling(cpu);
1185                 }
1186                 goto start_all_exit;
1187         }
1188         hws_state = HWS_STARTED;
1189         rc = 0;
1190
1191 start_all_exit:
1192         mutex_unlock(&hws_sem);
1193
1194         if (rc)
1195                 return rc;
1196
1197         register_oom_notifier(&hws_oom_notifier);
1198         hws_oom = 1;
1199         hws_flush_all = 0;
1200         /* now let them in, 1407 CPUMF external interrupts */
1201         ctl_set_bit(0, 5); /* set CR0 bit 58 */
1202
1203         return 0;
1204 }
1205
1206 /**
1207  * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1208  *
1209  * Returns 0 on success, !0 on failure.
1210  */
1211 int hwsampler_stop_all(void)
1212 {
1213         int tmp_rc, rc, cpu;
1214         struct hws_cpu_buffer *cb;
1215
1216         mutex_lock(&hws_sem);
1217
1218         rc = 0;
1219         if (hws_state == HWS_INIT) {
1220                 mutex_unlock(&hws_sem);
1221                 return rc;
1222         }
1223         hws_state = HWS_STOPPING;
1224         mutex_unlock(&hws_sem);
1225
1226         for_each_online_cpu(cpu) {
1227                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1228                 cb->stop_mode = 1;
1229                 tmp_rc = stop_sampling(cpu);
1230                 if (tmp_rc)
1231                         rc = tmp_rc;
1232         }
1233
1234         if (hws_wq)
1235                 flush_workqueue(hws_wq);
1236
1237         mutex_lock(&hws_sem);
1238         if (hws_oom) {
1239                 unregister_oom_notifier(&hws_oom_notifier);
1240                 hws_oom = 0;
1241         }
1242         hws_state = HWS_STOPPED;
1243         mutex_unlock(&hws_sem);
1244
1245         return rc;
1246 }