HPMC User Guide v 1.00
© 2022 Bassem W. Jamaleddine


2-4

   Matrix Multiplication Offloading to MIC

This chapter shows how to use your HPMC calculator to multiply matrices using openMP. The operations will be offloaded to the Intel Xeon Phi MIC.

Consider the program AxB.c shown in the following listing:

    --  Program Code 2.4.1 :      [LISTING AxB.c] - [Matrix Multiplication using openMP]
(raw text)
1.     #ifndef MIC_DEV
2.     #define MIC_DEV 0
3.     #endif
4.      
5.     #include <stdio.h>
6.     #include <stdlib.h>
7.     #include <omp.h>
8.     #include <math.h>
9.     #include <sys/time.h>
10.    
11.    double getTime(void) {
12.    struct timeval t;
13.    gettimeofday(&t,0);
14.    return ((double)t.tv_sec + ((double)t.tv_usec / 1000000.0));
15.    }
16.    
17.    // ################################ 
18.    // openMP multiply matrices
19.    void doMult(int size, float (* restrict A)[size],
20.         float (* restrict B)[size], float (* restrict C)[size]
21.        ) {
22.    
23.    #pragma offload target(mic:MIC_DEV) \
24.        in(A:length(size*size)) in( B:length(size*size))    \
25.        out(C:length(size*size))
26.        {
27.    #pragma omp parallel for default(none) shared(C,size)
28.            for (int i = 0; i < size; ++i)
29.                for (int j = 0; j < size; ++j)
30.                    C[i][j] =0.f;
31.            // matrix multiplication.
32.    #pragma omp parallel for default(none) shared(A,B,C,size)
33.            for (int i = 0; i < size; ++i)
34.                for (int k = 0; k < size; ++k)
35.                    for (int j = 0; j < size; ++j)
36.                      C[i][j] += A[i][k] * B[k][j];
37.        }
38.    }
39.     
40.    // ################################ 
41.    int main(int argc, char *argv[]) {
42.        if(argc != 4) {
43.            fprintf(stderr,"Use: %s size nThreads iter\n",argv[0]);
44.            return -1;
45.        }
46.        int i,j,k;
47.        int size=atoi(argv[1]);
48.        int nThreads=atoi(argv[2]);
49.        int iter=atoi(argv[3]);
50.        omp_set_num_threads(nThreads);
51.        float (*restrict A)[size] = malloc(sizeof(float)*size*size);
52.        float (*restrict B)[size] = malloc(sizeof(float)*size*size);
53.        float (*restrict C)[size] = malloc(sizeof(float)*size*size);
54.        // init natrices A and B 
55.    #pragma omp parallel for default(none) shared(A,B,size) private(i,j,k)
56.        for (i = 0; i < size; ++i) {
57.            for (j = 0; j < size; ++j) {
58.                A[i][j] = (float)((rand() % 3)-1);
59.                B[i][j] = (float)((rand() % 3)-1);
60.            }
61.        }
62.        // warmup
63.        doMult(size, A,B,C);
64.        double startTime = getTime();
65.        for (int i=0; i < iter; i++) {
66.            doMult(size, A,B,C);
67.        }
68.        double endTime = getTime();
69.        double lapsed = endTime-startTime;
70.        lapsed /= iter;
71.      
72.        printf("%s nThrds %d size %d elapsedtime %g GFlop/s %g\n",
73.             argv[0], nThreads, size, lapsed, 2e-9*size*size*size/lapsed);
74.     
75.        free(A); free(B); free(C);
76.        return 0;
77.    }
78.    

HPMC 2022


Compile AxB.c to be executed on the MIC:

icc -O3 -mmic -qopenmp -L /opt/intel/lib/mic -Wno-unknown-pragmas -std=c99 -liomp5 AxB.c -o AxB.mic



The following script will run the pogram in a loop, incrementing the number of threads by 24 in each iteration up to a maximum of 340 threads.
#!/bin/ksh
export KMP_AFFINITY="granularity=thread,balanced"
i=24
while [ $i -lt 340 ]
do
   xphi AxB.mic -a "1000 $i 10"
   let i+=24;
done


The coi (coprocessor offload infrastructure) processes as the program is executing.

03:30 root@HPMC9: /mm03fs/DOBB # ./AxB.mic.sh

03:30 root@HPMC9: /mm03fs/DOBB #  ./AxB.mic.sh 
/tmp/coi_procs/1/23650/AxB.mic nThrds 24 size 1000 elapsedtime 0.0400838 GFlop/s 49.8955
/tmp/coi_procs/1/23685/AxB.mic nThrds 48 size 1000 elapsedtime 0.0269411 GFlop/s 74.236
/tmp/coi_procs/1/23743/AxB.mic nThrds 72 size 1000 elapsedtime 0.027996 GFlop/s 71.4388
/tmp/coi_procs/1/23825/AxB.mic nThrds 96 size 1000 elapsedtime 0.0311322 GFlop/s 64.2422
/tmp/coi_procs/1/23931/AxB.mic nThrds 120 size 1000 elapsedtime 0.0186215 GFlop/s 107.403
/tmp/coi_procs/1/24060/AxB.mic nThrds 144 size 1000 elapsedtime 0.0262353 GFlop/s 76.2332
/tmp/coi_procs/1/24215/AxB.mic nThrds 168 size 1000 elapsedtime 0.0202382 GFlop/s 98.823
/tmp/coi_procs/1/24393/AxB.mic nThrds 192 size 1000 elapsedtime 0.0206483 GFlop/s 96.8603
/tmp/coi_procs/1/24595/AxB.mic nThrds 216 size 1000 elapsedtime 0.0144107 GFlop/s 138.786
/tmp/coi_procs/1/24822/AxB.mic nThrds 240 size 1000 elapsedtime 0.0197546 GFlop/s 101.242
/tmp/coi_procs/1/25071/AxB.mic nThrds 264 size 1000 elapsedtime 0.0258059 GFlop/s 77.5017
/tmp/coi_procs/1/25345/AxB.mic nThrds 288 size 1000 elapsedtime 0.0331669 GFlop/s 60.3011
/tmp/coi_procs/1/25643/AxB.mic nThrds 312 size 1000 elapsedtime 0.0454103 GFlop/s 44.0429
/tmp/coi_procs/1/25965/AxB.mic nThrds 336 size 1000 elapsedtime 0.0446899 GFlop/s 44.7528


On the MIC you can see the processes being executed as shown below.

(HPMC9) 03:21 root@mic0: ~ # pstree -up

(HPMC9) 03:21 root@mic0: ~ #  pstree -up 
init(1)-+-coi_daemon(4354,micuser)-+-AxB.mic(25071)-+-{AxB.mic}(25077)
        |                          |                |-{AxB.mic}(25078)
        |                          |                |-{AxB.mic}(25079)
        |                          |                |-{AxB.mic}(25080)
        |                          |                |-{AxB.mic}(25081)
        |                          |                |-{AxB.mic}(25082)

 ... 

        |                          |                |-{AxB.mic}(25335)
        |                          |                |-{AxB.mic}(25336)
        |                          |                |-{AxB.mic}(25337)
        |                          |                |-{AxB.mic}(25338)
        |                          |                |-{AxB.mic}(25339)
        |                          |                `-{AxB.mic}(25340)
        |                          |-{coi_daemon}(4356)
        |                          `-{coi_daemon}(25073)
        |-getty(4370)
        |-klogd(4333)
        |-mpssd(4339)-+-{mpssd}(4341)
        |             `-{mpssd}(4342)
        |-portmap(4290,1)
        |-sshd(4322)-+-sshd(4382)---bash(4384)---dint(23639)---cat(25342)
        |            |-sshd(13638)---bash(13640)---dcpus(23643)
        |            `-sshd(13642)---bash(13644)---pstree(25341)
        |-syslogd(4330)
        `-udevd(4113)-+-udevd(4336)
                      `-udevd(4363)

■ Interrupt and Stat Activities of a Program Offloaded to MIC


As the program is executed in a loop, and it is being offloaded to the MIC, we can reveal the stats of the cores. We use dcpus to display the stats by looping 24 times with a delay of 3 seconds in between. Since MIC has 228 cores, we use the -prune 6,3 option to display only the first six cores and the last three cores. Notice how the system and user usage went from 0 to higher numbers on the top most cores. As the loop is running, and more threads are allocated, the system usage is also increased on tne upper cores.

(HPMC9) 03:30 root@mic0: /tools # ./dcpus -transpose -iter 24 -delay 3 -prune 6,3 -stats user,system,idle,softirq

This command shows the specific stats (user,system,idle,softirq) on the first six cores ans last three cores (-prune 6,3) that are being transposed. The command is run in a loop iterating 24 times.
(HPMC9) 03:30 root@mic0: /tools #   ./dcpus -transpose -iter 24 -delay 3 -prune 6,3 -stats user,system,idle,softirq 
                                    user    system      idle   softirq 
2026-04-26 03:30:57       CPU0         0         0       141         0 
2026-04-26 03:30:57       CPU1         0         0       194         1 
2026-04-26 03:30:57       CPU2         0         0       191         0 
2026-04-26 03:30:57       CPU3         0         0       192         0 
2026-04-26 03:30:57       CPU4         0         0       202         0 
2026-04-26 03:30:57       CPU5         0         0       192         0 
                           ...
2026-04-26 03:30:57     CPU225         0         0        98         0 
2026-04-26 03:30:57     CPU226         0         0        99         0 
2026-04-26 03:30:57     CPU227         0         0       113         0 
                        summed         3        39     33396        17 
used=59  idle=33396  usage 0.1763562995068%
                                    user    system      idle   softirq 
2026-04-26 03:31:00       CPU0         0         0       344         0 
2026-04-26 03:31:00       CPU1         0         0       299         1 
2026-04-26 03:31:00       CPU2         0         0       306         0 
2026-04-26 03:31:00       CPU3         0         0       303         0 
2026-04-26 03:31:00       CPU4         0         0       299         0 
2026-04-26 03:31:00       CPU5         0         0       306         0 
                           ...
2026-04-26 03:31:00     CPU225         0         0       399         0 
2026-04-26 03:31:00     CPU226         0         0       391         0 
2026-04-26 03:31:00     CPU227         0         0       388         0 
                        summed        13        84     78386        45 
used=142  idle=78386  usage 0.180827220863896%
                                    user    system      idle   softirq 
2026-04-26 03:31:03       CPU0         0         1       349         0 
2026-04-26 03:31:03       CPU1         2       231       164         0 
2026-04-26 03:31:03       CPU2         0         0       394         0 
2026-04-26 03:31:03       CPU3         0         0       403         0 
2026-04-26 03:31:03       CPU4         0         0       404         0 
2026-04-26 03:31:03       CPU5         5       225       172         0 
                           ...
2026-04-26 03:31:03     CPU225         0        80       238         0 
2026-04-26 03:31:03     CPU226         0        41       284         0 
2026-04-26 03:31:03     CPU227         1         2       305         0 
                        summed       125      5630     74393        34 
used=5789  idle=74393  usage 7.21982489835624%
                                    user    system      idle   softirq 
2026-04-26 03:31:07       CPU0         0        35       307         4 
2026-04-26 03:31:07       CPU1        73       104       128         0 
2026-04-26 03:31:07       CPU2         0         0       298         1 
2026-04-26 03:31:07       CPU3         0         0       297         0 
2026-04-26 03:31:07       CPU4         0         0       297         0 
2026-04-26 03:31:07       CPU5        55       126       119         0 
                           ...
2026-04-26 03:31:07     CPU225         0        80       315         0 
2026-04-26 03:31:07     CPU226         0        40       355         0 
2026-04-26 03:31:07     CPU227         0         0       414         0 
                        summed      1745      2709     72220        39 
used=4493  idle=72220  usage 5.8568951807386%
                                    user    system      idle   softirq 
2026-04-26 03:31:10       CPU0         0       125       222         0 
2026-04-26 03:31:10       CPU1        10       293        85         0 
2026-04-26 03:31:10       CPU2         0         1       309         1 
2026-04-26 03:31:10       CPU3         0         0       307         0 
2026-04-26 03:31:10       CPU4         0         0       307         0 
2026-04-26 03:31:10       CPU5        11       284        99         0 
                           ...
2026-04-26 03:31:10     CPU225         0         1       346         0 
2026-04-26 03:31:10     CPU226         0         0       340         2 
2026-04-26 03:31:10     CPU227         0         0       284         0 
                        summed       683     13723     67670        19 
used=14425  idle=67670  usage 17.5711066447409%
                                    user    system      idle   softirq 
2026-04-26 03:31:14       CPU0         0        45       306         0 
2026-04-26 03:31:14       CPU1        34       170       147         0 
2026-04-26 03:31:14       CPU2         0       168       268         0 
2026-04-26 03:31:14       CPU3         0         0       402         0 
2026-04-26 03:31:14       CPU4         0         0       391         0 
2026-04-26 03:31:14       CPU5        34       166       150         1 
                           ...
2026-04-26 03:31:14     CPU225         0         2       349         0 
2026-04-26 03:31:14     CPU226         0        81       253         1 
2026-04-26 03:31:14     CPU227         0         0       401         0 
                        summed      1575     12232     67384        28 
used=13841  idle=67384  usage 17.0403200984918%
                                    user    system      idle   softirq 
2026-04-26 03:31:17       CPU0         0       113       234         2 
2026-04-26 03:31:17       CPU1        52       155        81         0 
2026-04-26 03:31:17       CPU2        47       160        55         0 
2026-04-26 03:31:17       CPU3         0         0       299         0 
2026-04-26 03:31:17       CPU4         0         0       310         0 
2026-04-26 03:31:17       CPU5        48       156        42         0 
                           ...
2026-04-26 03:31:17     CPU225         9         0       340         0 
2026-04-26 03:31:17     CPU226         1        88       279         2 
2026-04-26 03:31:17     CPU227         0         0       362         0 
                        summed      3167     11943     60548        13 
used=15123  idle=60548  usage 19.9851990855149%
                                    user    system      idle   softirq 
2026-04-26 03:31:21       CPU0         0       127       221         0 
2026-04-26 03:31:21       CPU1         3       296       107         0 
2026-04-26 03:31:21       CPU2         4       292       137         0 
2026-04-26 03:31:21       CPU3         0         0       395         0 
2026-04-26 03:31:21       CPU4         0         0       395         0 
2026-04-26 03:31:21       CPU5         1       294       151         0 
                           ...
2026-04-26 03:31:21     CPU225         0         2       343         0 
2026-04-26 03:31:21     CPU226         8         0       338         0 
2026-04-26 03:31:21     CPU227         0        48       289         0 
                        summed       572     28079     55538         9 
used=28660  idle=55538  usage 34.0388132734744%
                                    user    system      idle   softirq 
2026-04-26 03:31:24       CPU0         0       126       229         0 
2026-04-26 03:31:24       CPU1        51       132       173         0 
2026-04-26 03:31:24       CPU2        53       123       179         0 
2026-04-26 03:31:24       CPU3         2       122       273         0 
2026-04-26 03:31:24       CPU4         0         0       396         0 
2026-04-26 03:31:24       CPU5        38       137       179         0 
                           ...
2026-04-26 03:31:24     CPU225         0         0       358         0 
2026-04-26 03:31:24     CPU226         0         2       355         0 
2026-04-26 03:31:24     CPU227         0         0       335         0 
                        summed      4381     15568     62536        11 
used=20015  idle=62536  usage 24.2456178604741%
                                    user    system      idle   softirq 
2026-04-26 03:31:28       CPU0         0       156       192         1 
2026-04-26 03:31:28       CPU1        37       167       143         0 
2026-04-26 03:31:28       CPU2        56       133       160         0 
2026-04-26 03:31:28       CPU3        49       139       160         0 
2026-04-26 03:31:28       CPU4         0         0       329         0 
2026-04-26 03:31:28       CPU5        47       141       160         0 
                           ...
2026-04-26 03:31:28     CPU225        15         1       334         0 
2026-04-26 03:31:28     CPU226         0        89       261         0 
2026-04-26 03:31:28     CPU227         0         4       377         0 
                        summed      4855     18524     58617         9 
used=23464  idle=58617  usage 28.5863963645667%
                                    user    system      idle   softirq 
2026-04-26 03:31:31       CPU0        16        49       284         0 
2026-04-26 03:31:31       CPU1        69       277         2         0 
2026-04-26 03:31:31       CPU2        61       260        28         0 
2026-04-26 03:31:31       CPU3        94       253         2         0 
2026-04-26 03:31:31       CPU4         0         0       367         1 
2026-04-26 03:31:31       CPU5        45       276        27         0 
                           ...
2026-04-26 03:31:31     CPU225         0        40       309         0 
2026-04-26 03:31:31     CPU226         1        41       307         0 
2026-04-26 03:31:31     CPU227         0        87       259         0 
                        summed      4274     44741     28527        20 
used=49094  idle=28527  usage 63.2483477409464%
                                    user    system      idle   softirq 
2026-04-26 03:31:35       CPU0         6        42       305         0 
2026-04-26 03:31:35       CPU1        24       135       194         0 
2026-04-26 03:31:35       CPU2        24       126       203         0 
2026-04-26 03:31:35       CPU3        23       127       203         0 
2026-04-26 03:31:35       CPU4         0         0       305         0 
2026-04-26 03:31:35       CPU5        24       128       201         0 
                           ...
2026-04-26 03:31:35     CPU225         1        81       270         0 
2026-04-26 03:31:35     CPU226         0        43       295         0 
2026-04-26 03:31:35     CPU227         2         0       344         9 
                        summed      3343     21450     56278        18 
used=24811  idle=56278  usage 30.5972450024048%
                                    user    system      idle   softirq 
2026-04-26 03:31:38       CPU0        11        42       298         0 
2026-04-26 03:31:38       CPU1        69       159        65         0 
2026-04-26 03:31:38       CPU2        64       162       125         0 
2026-04-26 03:31:38       CPU3        59       168       121         0 
2026-04-26 03:31:38       CPU4         0         0       401         0 
2026-04-26 03:31:38       CPU5        35       190       123         0 
                           ...
2026-04-26 03:31:38     CPU225         0        42       310         0 
2026-04-26 03:31:38     CPU226         7        46       313         0 
2026-04-26 03:31:38     CPU227         1       117       234         0 
                        summed      6534     31213     35642         8 
used=37930  idle=35642  usage 51.5549393791116%
                                    user    system      idle   softirq 
2026-04-26 03:31:42       CPU0         0        81       267         1 
2026-04-26 03:31:42       CPU1         6       293       105         0 
2026-04-26 03:31:42       CPU2        62       191        85         0 
2026-04-26 03:31:42       CPU3        31       259        60         0 
2026-04-26 03:31:42       CPU4        44       235        59         0 
2026-04-26 03:31:42       CPU5         1       289        59         0 
                           ...
2026-04-26 03:31:42     CPU225        10         2       316         0 
2026-04-26 03:31:42     CPU226         0        48       299         0 
2026-04-26 03:31:42     CPU227         0        45       283         0 
                        summed      1263     54167     31293        12 
used=55442  idle=31293  usage 63.921139101862%
                                    user    system      idle   softirq 
2026-04-26 03:31:45       CPU0         2       123       227         0 
2026-04-26 03:31:45       CPU1        68       116       167         0 
2026-04-26 03:31:45       CPU2        24       103       235         0 
2026-04-26 03:31:45       CPU3        66       105       183         0 
2026-04-26 03:31:45       CPU4        22       104       236         0 
2026-04-26 03:31:45       CPU5        42       130       179         0 
                           ...
2026-04-26 03:31:45     CPU225         0         1       369         2 
2026-04-26 03:31:45     CPU226         0         2       351         0 
2026-04-26 03:31:45     CPU227         0         0       305         0 
                        summed      7521     26572     47220        12 
used=34184  idle=47220  usage 41.993022455899%
                                    user    system      idle   softirq 
2026-04-26 03:31:49       CPU0        14       122       214         0 
2026-04-26 03:31:49       CPU1        20       211        74         0 
2026-04-26 03:31:49       CPU2        24       208       118         0 
2026-04-26 03:31:49       CPU3        33       198        74         0 
2026-04-26 03:31:49       CPU4        56       176        62         0 
2026-04-26 03:31:49       CPU5        21       211       118         0 
                           ...
2026-04-26 03:31:49     CPU225         1         0       345         0 
2026-04-26 03:31:49     CPU226         1        88       240         0 
2026-04-26 03:31:49     CPU227         0         1       399         0 
                        summed      8312     39342     21404         2 
used=47776  idle=21404  usage 69.0604220873085%
                                    user    system      idle   softirq 
2026-04-26 03:31:52       CPU0         9        85       254         0 
2026-04-26 03:31:52       CPU1        24       268       100         0 
2026-04-26 03:31:52       CPU2         8       168       172         0 
2026-04-26 03:31:52       CPU3         9       208       175         0 
2026-04-26 03:31:52       CPU4         8       218       176         0 
2026-04-26 03:31:52       CPU5        19       242        87         0 
                           ...
2026-04-26 03:31:52     CPU225         0         2       350         0 
2026-04-26 03:31:52     CPU226         0       130       220         0 
2026-04-26 03:31:52     CPU227         0         0       354        12 
                        summed      2215     45485     42446        19 
used=47735  idle=42446  usage 52.9324358789545%
                                    user    system      idle   softirq 
2026-04-26 03:31:56       CPU0         0       134       193         8 
2026-04-26 03:31:56       CPU1         4       215       128         0 
2026-04-26 03:31:56       CPU2         0       202       143         0 
2026-04-26 03:31:56       CPU3         0       188       154         0 
2026-04-26 03:31:56       CPU4         0       165       150         0 
2026-04-26 03:31:56       CPU5         1       202       146         0 
                           ...
2026-04-26 03:31:56     CPU225         9        40       300         0 
2026-04-26 03:31:56     CPU226        14        21       331         0 
2026-04-26 03:31:56     CPU227         0        49       228         0 
                        summed       136     41654     35033        46 
used=41836  idle=35033  usage 54.4250608177549%
                                    user    system      idle   softirq 
2026-04-26 03:31:59       CPU0         1        42       303        16 
2026-04-26 03:31:59       CPU1        27       160       166         0 
2026-04-26 03:31:59       CPU2         8       107       239         0 
2026-04-26 03:31:59       CPU3        10       105       242         0 
2026-04-26 03:31:59       CPU4         9       104       270         0 
2026-04-26 03:31:59       CPU5        26       116       208         0 
                           ...
2026-04-26 03:31:59     CPU225         8        46       293         0 
2026-04-26 03:31:59     CPU226         4         1       346         1 
2026-04-26 03:31:59     CPU227         0        40       382         0 
                        summed      2824     24286     55308        32 
used=27150  idle=55308  usage 32.9258531616095%
                                    user    system      idle   softirq 
2026-04-26 03:32:03       CPU0         8        45       294         5 
2026-04-26 03:32:03       CPU1        36       144       174         0 
2026-04-26 03:32:03       CPU2        32       110       209         0 
2026-04-26 03:32:03       CPU3        11        84       258         0 
2026-04-26 03:32:03       CPU4        25        57       271         0 
2026-04-26 03:32:03       CPU5        43       110       200         0 
                           ...
2026-04-26 03:32:03     CPU225         0       121       227         4 
2026-04-26 03:32:03     CPU226         0         2       348         3 
2026-04-26 03:32:03     CPU227         1         0       330         0 
                        summed      4861     16626     58879        50 
used=21580  idle=58879  usage 26.8211138592327%
                                    user    system      idle   softirq 
2026-04-26 03:32:06       CPU0        10        42       253        36 
2026-04-26 03:32:06       CPU1        37       221        65         0 
2026-04-26 03:32:06       CPU2        33       204       112         0 
2026-04-26 03:32:06       CPU3        23       140       178         1 
2026-04-26 03:32:06       CPU4        26       126       197         0 
2026-04-26 03:32:06       CPU5        35       194       121         0 
                           ...
2026-04-26 03:32:06     CPU225        10        41       258         4 
2026-04-26 03:32:06     CPU226         0       134       214         0 
2026-04-26 03:32:06     CPU227         0         0       317         6 
                        summed      6416     37244     28776        63 
used=43724  idle=28776  usage 60.3089655172414%
                                    user    system      idle   softirq 
2026-04-26 03:32:10       CPU0         9       130       199         9 
2026-04-26 03:32:10       CPU1        46       211       119         0 
2026-04-26 03:32:10       CPU2        39       207       104         0 
2026-04-26 03:32:10       CPU3        20       141       192         0 
2026-04-26 03:32:10       CPU4        22       179       146         0 
2026-04-26 03:32:10       CPU5        45       206        98         0 
                           ...
2026-04-26 03:32:10     CPU225         0         1       362         6 
2026-04-26 03:32:10     CPU226         0         0       314        16 
2026-04-26 03:32:10     CPU227         0        50       328         4 
                        summed      6301     39081     40555        58 
used=45514  idle=40555  usage 52.8808281727451%
                                    user    system      idle   softirq 
2026-04-26 03:32:13       CPU0         0        79       238         0 
2026-04-26 03:32:13       CPU1         2        12       249         1 
2026-04-26 03:32:13       CPU2         0         1       321         0 
2026-04-26 03:32:13       CPU3         0         0       265         0 
2026-04-26 03:32:13       CPU4         0         0       350         0 
2026-04-26 03:32:13       CPU5         8         1       334         0 
                           ...
2026-04-26 03:32:13     CPU225         0         0       322         0 
2026-04-26 03:32:13     CPU226         0         0       350         0 
2026-04-26 03:32:13     CPU227         0         0       295         0 
                        summed        16       186     69814        22 
used=224  idle=69814  usage 0.319826379965162%
                                    user    system      idle   softirq 
2026-04-26 03:32:17       CPU0         0         0       354         0 
2026-04-26 03:32:17       CPU1         0         0       398         1 
2026-04-26 03:32:17       CPU2         0         0       332         0 
2026-04-26 03:32:17       CPU3         0         0       390         0 
2026-04-26 03:32:17       CPU4         0         0       298         0 
2026-04-26 03:32:17       CPU5         0         0       301         3 
                           ...
2026-04-26 03:32:17     CPU225         0         0       367         0 
2026-04-26 03:32:17     CPU226         0         0       345         0 
2026-04-26 03:32:17     CPU227         0         0       401         0 
                        summed        16        86     79106        38 
used=140  idle=79106  usage 0.17666506826843%




On the MIC we can reveal the interrupt activities as the the offloaded program is being run. As the program is executed in a loop, and it is being offloaded to the MIC, we can reveal the interrupt activities of the MIC. We use dint to display the interrupts (LOC,TLB,RES,TRM,CAL) by looping 24 times with a delay of 3 seconds in betwee. Since MIC has 228 cores, we use the -prune 6,3 option to display only the first six cores and the last three cores. Notice how the system interrupts of RES,LOC are being increased on the top most cores. As the loop is running, and more threads are allocated, the system interrupts also increased on tne upper cores.

(HPMC9) 03:30 root@mic0: /tools # ./dint -transpose -iter 24 -delay 3 -prune 6,3 -transpose -interrupts LOC,TLB,RES,TRM,CAL -sortby RES

#  ./dint -transpose -iter 24 -delay 3 -prune 6,3 -transpose -interrupts LOC,TLB,RES,TRM,CAL -sortby RES 
                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:30:54   CPU30        53         0        10         0         0 
2026-04-26 03:30:54   CPU29        53         0         6         0         0 
2026-04-26 03:30:54    CPU0        33         0         2         0         0 
2026-04-26 03:30:54  CPU222        12         0         1         0         0 
2026-04-26 03:30:54  CPU225        13         0         1         0         0 
2026-04-26 03:30:54   CPU26        54         1         1         0         0 
                        ...
2026-04-26 03:30:54   CPU90        12         0         0         0         0 
2026-04-26 03:30:54   CPU32        12         0         0         0         0 
2026-04-26 03:30:54  CPU127        12         0         0         0         0 
                     summed      2994         2        24         0        21 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:30:57   CPU38        70         0        11         0         0 
2026-04-26 03:30:57   CPU54       102         1        10         0         0 
2026-04-26 03:30:57   CPU62        91         0         9         0         0 
2026-04-26 03:30:57  CPU225        30         0         8         0         0 
2026-04-26 03:30:57  CPU226        27         0         8         0         0 
2026-04-26 03:30:57    CPU0        79         0         7         0         0 
                        ...
2026-04-26 03:30:57   CPU90        29         0         0         0         0 
2026-04-26 03:30:57   CPU32        30         0         0         0         0 
2026-04-26 03:30:57  CPU127        29         0         0         0         0 
                     summed      7765         9        80         0        23 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:01  CPU226        79         2        25         0         3 
2026-04-26 03:31:01    CPU0       121         2        24         0         0 
2026-04-26 03:31:01  CPU225        64        57        22         0         3 
2026-04-26 03:31:01  CPU227        80         1        22         0         3 
2026-04-26 03:31:01    CPU6        78         1        10         0         3 
2026-04-26 03:31:01   CPU78        71         0         9         0         3 
                        ...
2026-04-26 03:31:01  CPU190        29         0         0         0         3 
2026-04-26 03:31:01  CPU200        29         0         0         0         3 
2026-04-26 03:31:01  CPU206        29         0         0         0         3 
                     summed      7510        72       314         0       715 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:04   CPU21       344         2      4449         0         0 
2026-04-26 03:31:04   CPU29       346         2      4396         0         0 
2026-04-26 03:31:04   CPU13       344         2      4391         0         0 
2026-04-26 03:31:04   CPU89       339         2      4227         0         0 
2026-04-26 03:31:04   CPU45       348         1      4189         0         0 
2026-04-26 03:31:04   CPU41       344         2      4167         0         0 
                        ...
2026-04-26 03:31:04   CPU90        29         1         0         0         0 
2026-04-26 03:31:04   CPU32        29         1         0         0         0 
2026-04-26 03:31:04  CPU127        28         1         0         0         0 
                     summed     14682       317     95269         0       172 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:08   CPU21       173         9      2166         0         8 
2026-04-26 03:31:08   CPU29       175         9      2122         0         8 
2026-04-26 03:31:08   CPU17       174         9      2115         0         8 
2026-04-26 03:31:08   CPU13       171         9      1791         0         8 
2026-04-26 03:31:08    CPU5       174         8      1585         0         8 
2026-04-26 03:31:08   CPU25       174         8      1485         0         8 
                        ...
2026-04-26 03:31:08   CPU90        29         1         1         0         8 
2026-04-26 03:31:08   CPU32        28         1         1         0         8 
2026-04-26 03:31:08  CPU127        28         1         1         0         8 
                     summed     12909       568     31386         0      1914 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:11   CPU33       284         8      1661         0         2 
2026-04-26 03:31:11   CPU97       283         6      1625         0         2 
2026-04-26 03:31:11   CPU53       283         8      1616         0         2 
2026-04-26 03:31:11   CPU73       279         7      1607         0         2 
2026-04-26 03:31:11  CPU113       284         7      1562         0         2 
2026-04-26 03:31:11  CPU105       285         8      1558         0         2 
                        ...
2026-04-26 03:31:11  CPU118        29         0         0         0         2 
2026-04-26 03:31:11   CPU90        29         0         0         0         2 
2026-04-26 03:31:11  CPU127        29         0         0         0         2 
                     summed     19617       372     60225         0       590 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:15  CPU161       302         2      1324         0         6 
2026-04-26 03:31:15    CPU1       333        72      1322         0         5 
2026-04-26 03:31:15  CPU165       312         2      1312         0         5 
2026-04-26 03:31:15  CPU197       309         2      1310         0         5 
2026-04-26 03:31:15  CPU157       309         2      1305         0         5 
2026-04-26 03:31:15  CPU173       307         2      1300         0         5 
                        ...
2026-04-26 03:31:15   CPU90        29         1         1         0         5 
2026-04-26 03:31:15   CPU32        30         1         1         0         5 
2026-04-26 03:31:15  CPU127        29         1         1         0         5 
                     summed     27565       559     83410         0      1429 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:18    CPU6       228       107       660         0         8 
2026-04-26 03:31:18  CPU173       191        10       646         0         8 
2026-04-26 03:31:18  CPU161       192        10       642         0         8 
2026-04-26 03:31:18  CPU105       193         9       638         0         8 
2026-04-26 03:31:18    CPU1       279        99       628         0         8 
2026-04-26 03:31:18  CPU181       194        11       621         0         8 
                        ...
2026-04-26 03:31:18  CPU206        29         1         1         0         8 
2026-04-26 03:31:18   CPU32        29         1         1         0         8 
2026-04-26 03:31:18  CPU127        29         1         1         0         8 
                     summed     21718      1226     28897         0      1988 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:22   CPU45       272        13       937         0         2 
2026-04-26 03:31:22   CPU41       260        12       934         0         2 
2026-04-26 03:31:22   CPU61       271        14       923         0         2 
2026-04-26 03:31:22   CPU53       268        11       880         0         2 
2026-04-26 03:31:22   CPU33       271        12       871         0         2 
2026-04-26 03:31:22   CPU25       272        12       832         0         2 
                        ...
2026-04-26 03:31:22  CPU206        29         0         0         0         2 
2026-04-26 03:31:22   CPU32        30         0         0         0         2 
2026-04-26 03:31:22  CPU127        29         0         0         0         2 
                     summed     30634      1244     62036         0       708 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:26    CPU1       347       121      1013         0         6 
2026-04-26 03:31:26    CPU6       337       124       821         0         6 
2026-04-26 03:31:26   CPU10       298        92       820         0         6 
2026-04-26 03:31:26  CPU105       285         2       762         0         6 
2026-04-26 03:31:26   CPU69       284         2       726         0         6 
2026-04-26 03:31:26  CPU102       287         2       717         0         6 
                        ...
2026-04-26 03:31:26   CPU71        28         1         1         0         6 
2026-04-26 03:31:26   CPU32        28         1         1         0         6 
2026-04-26 03:31:26  CPU127        28         1         1         0         6 
                     summed     37988       693     52706         0      1645 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:29    CPU1       274       146       747         0         7 
2026-04-26 03:31:29  CPU109       237        10       664         0         7 
2026-04-26 03:31:29   CPU10       250       128       605         0         7 
2026-04-26 03:31:29    CPU6       249       160       582         0         7 
2026-04-26 03:31:29  CPU105       237        10       553         0         7 
2026-04-26 03:31:29  CPU125       239        12       446         0         7 
                        ...
2026-04-26 03:31:29   CPU16        29         1         1         0         7 
2026-04-26 03:31:29  CPU200        29         1         1         0         7 
2026-04-26 03:31:29   CPU32        29         1         1         0         7 
                     summed     36162      2072     31776         0      1739 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:33  CPU102       264         8       612         0         7 
2026-04-26 03:31:33  CPU165       265         7       558         0         7 
2026-04-26 03:31:33  CPU185       265         8       546         0         7 
2026-04-26 03:31:33  CPU161       264         8       536         0         7 
2026-04-26 03:31:33  CPU198       265         7       518         0         7 
2026-04-26 03:31:33   CPU90       292         8       510         0         7 
                        ...
2026-04-26 03:31:33  CPU163        32         0         0         0         7 
2026-04-26 03:31:33  CPU220        32         0         0         0         7 
2026-04-26 03:31:33  CPU200        32         0         0         0         7 
                     summed     41379      1387     49359         0      1838 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:37    CPU1       348       167       907         0         0 
2026-04-26 03:31:37   CPU10       349       166       767         0         0 
2026-04-26 03:31:37  CPU137       338         2       674         0         0 
2026-04-26 03:31:37  CPU145       338         2       671         0         0 
2026-04-26 03:31:37  CPU214       338         3       656         0         0 
2026-04-26 03:31:37  CPU222       339         2       621         0         0 
                        ...
2026-04-26 03:31:37   CPU44        29         1         0         0         0 
2026-04-26 03:31:37   CPU16        29         1         0         0         0 
2026-04-26 03:31:37   CPU32        30         1         0         0         0 
                     summed     58745       884     71948         0       324 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:40    CPU1       292       194       708         0         8 
2026-04-26 03:31:40   CPU10       278       199       656         0         8 
2026-04-26 03:31:40    CPU6       290       139       382         0         8 
2026-04-26 03:31:40   CPU68       178         2       278         0         8 
2026-04-26 03:31:40   CPU62       234        12       278         0         8 
2026-04-26 03:31:40   CPU63       234        12       277         0         8 
                        ...
2026-04-26 03:31:40  CPU148        28         1         1         0         8 
2026-04-26 03:31:40  CPU220        28         1         1         0         8 
2026-04-26 03:31:40  CPU200        28         1         1         0         8 
                     summed     45793      2878     25259         0      1988 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:44   CPU53       250        11       558         0         8 
2026-04-26 03:31:44   CPU85       250        11       531         0         8 
2026-04-26 03:31:44  CPU141       251        11       513         0         8 
2026-04-26 03:31:44  CPU153       248         6       501         0         8 
2026-04-26 03:31:44  CPU137       250        11       469         0         8 
2026-04-26 03:31:44  CPU129       251         7       452         0         8 
                        ...
2026-04-26 03:31:44  CPU148        29         0         0         0         8 
2026-04-26 03:31:44  CPU220        29         0         0         0         8 
2026-04-26 03:31:44  CPU200        29         0         0         0         8 
                     summed     48171      2297     49533         0      2032 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:47    CPU1       356       224       971         0         2 
2026-04-26 03:31:47   CPU10       355       222       959         0         2 
2026-04-26 03:31:47    CPU6       355       131       631         0         2 
2026-04-26 03:31:47   CPU14       354        96       550         0         2 
2026-04-26 03:31:47  CPU118       337        15       547         0         3 
2026-04-26 03:31:47    CPU2       357        15       485         0         2 
                        ...
2026-04-26 03:31:47  CPU204        29         1         1         0         2 
2026-04-26 03:31:47  CPU220        29         1         1         0         2 
2026-04-26 03:31:47  CPU200        29         1         1         0       204 
                     summed     71825      3748     58916         0       786 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:51    CPU1       310       227       575         0         5 
2026-04-26 03:31:51   CPU10       197       237       560         0         5 
2026-04-26 03:31:51    CPU6       262       180       490         0         5 
2026-04-26 03:31:51   CPU14       258        55       270         0         5 
2026-04-26 03:31:51   CPU34       208         2       203         0         5 
2026-04-26 03:31:51   CPU44       211         3       200         0         5 
                        ...
2026-04-26 03:31:51    CPU0       209         2        40         0         5 
2026-04-26 03:31:51  CPU226       160        60        26         0         4 
2026-04-26 03:31:51  CPU227       134         3        24         0       184 
                     summed     47564      1256     25065         0      1363 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:55    CPU1       314       243       932         0         8 
2026-04-26 03:31:55    CPU6       258       273       733         0         8 
2026-04-26 03:31:55   CPU10       285       242       565         0         8 
2026-04-26 03:31:55   CPU21       272         7       451         0         8 
2026-04-26 03:31:55   CPU53       271         7       423         0         8 
2026-04-26 03:31:55   CPU49       272         7       416         0         8 
                        ...
2026-04-26 03:31:55  CPU226       168         2        53         0        19 
2026-04-26 03:31:55  CPU227       179         2        47         0        54 
2026-04-26 03:31:55    CPU0       235        58        31         0        45 
                     summed     50492      2039     37130         0      1933 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:31:58    CPU1       267       306       930         0         7 
2026-04-26 03:31:58    CPU6       211       304       660         0         7 
2026-04-26 03:31:58   CPU10       217       277       656         0         7 
2026-04-26 03:31:58   CPU97       221         7       376         0         7 
2026-04-26 03:31:58   CPU77       218         7       317         0         7 
2026-04-26 03:31:58   CPU69       218         7       307         0         7 
                        ...
2026-04-26 03:31:58    CPU0       223         5        26         0        15 
2026-04-26 03:31:58  CPU226       130         2        26         0        30 
2026-04-26 03:31:58  CPU227       130         0        25         0         5 
                     summed     39673      2126     32380         0      1712 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:32:02    CPU1       299        20       475         0         4 
2026-04-26 03:32:02  CPU209       281         4       338         0         4 
2026-04-26 03:32:02   CPU93       274         4       328         0         4 
2026-04-26 03:32:02  CPU181       278         4       320         0         4 
2026-04-26 03:32:02   CPU69       274         4       316         0         4 
2026-04-26 03:32:02   CPU49       276         4       296         0         4 
                        ...
2026-04-26 03:32:02    CPU0       215         1        18         0        40 
2026-04-26 03:32:02  CPU226        87         1        17         0        54 
2026-04-26 03:32:02  CPU227       188         3        14         0        10 
                     summed     46214       763     37389         0      1107 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:32:05    CPU1       318       278       939         0         5 
2026-04-26 03:32:05    CPU6       338       331       787         0         5 
2026-04-26 03:32:05   CPU10       311       293       671         0         5 
2026-04-26 03:32:05   CPU33       279         7       362         0         5 
2026-04-26 03:32:05   CPU45       280         7       280         0         5 
2026-04-26 03:32:05   CPU49       282         7       271         0         5 
                        ...
2026-04-26 03:32:05  CPU225       204         6        36         0         4 
2026-04-26 03:32:05    CPU0       265         0        24         0         5 
2026-04-26 03:32:05  CPU227        64         2        18         0       166 
                     summed     53800      2270     40860         0      1335 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:32:09    CPU6       307       355       775         0         6 
2026-04-26 03:32:09    CPU1       265       282       674         0         7 
2026-04-26 03:32:09   CPU10       243       309       627         0         6 
2026-04-26 03:32:09   CPU21       238         3       233         0         6 
2026-04-26 03:32:09   CPU11       199         6       202         0         6 
2026-04-26 03:32:09   CPU61       237         4       194         0         6 
                        ...
2026-04-26 03:32:09  CPU226       172         2        31         0        27 
2026-04-26 03:32:09    CPU0       198         7        28         0        16 
2026-04-26 03:32:09  CPU227        97         3        25         0        79 
                     summed     46771      1607     28257         0      1508 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:32:13    CPU1       152        23       246         0         4 
2026-04-26 03:32:13  CPU129       121         4        90         0         4 
2026-04-26 03:32:13   CPU98        91         4        88         0         4 
2026-04-26 03:32:13  CPU162        91         4        83         0         4 
2026-04-26 03:32:13  CPU193       152         4        78         0         4 
2026-04-26 03:32:13   CPU34        93         4        75         0         4 
                        ...
2026-04-26 03:32:13  CPU203        63         3        21         0         4 
2026-04-26 03:32:13    CPU0       130         1         6         0         0 
2026-04-26 03:32:13  CPU227        69         0         4         0         4 
                     summed     19147       803      8377         0       932 

                                  LOC       TLB       RES       TRM       CAL 
2026-04-26 03:32:16   CPU14        71         0        10         0         0 
2026-04-26 03:32:16   CPU22        74         0         9         0         0 
2026-04-26 03:32:16   CPU13        71         0         6         0         0 
2026-04-26 03:32:16  CPU225        42         0         6         0         0 
2026-04-26 03:32:16  CPU226        58         0         6         0         0 
2026-04-26 03:32:16    CPU0        31         0         4         0         0 
                        ...
2026-04-26 03:32:16   CPU90        29         0         0         0         0 
2026-04-26 03:32:16   CPU32        29         0         0         0         0 
2026-04-26 03:32:16  CPU127        29         0         0         0         0 
                     summed      7068         4        53         0         8