Question

In MPI, I am doing a reduce operation(minimum) on a value. This works fine, but how do I grab the processor number that the minimum came from and solicit that processor for more information(or send the additional data with the reduce operation)?

Answer 1

If you don’t mind paring each value locally with an integer index (filled in this case with the value of the local rank), you can use the MPI_MINLOC or MPI_MAXLOC builtin operations for reduce; or it’s fairly easy to write your own MPI reduction operator to include things like multiple indices, etcc

Updated to add:
With the builtin operators MINLOC or MAXLOC, instead of passing in a single value to find the minimum of, you pass in that plus an integer index. That index can have any value you want it to, but it “follows” the other value along. MPI has built in “pair” data types – MPI_DOUBLE_INT for a double + an int, or MPI_2INT for two ints, that you can use.

So say you want to find the minimum of an integer array, and on which MPI task it was located. As normal, you find your local minimum on each task, and do the reduce; but this time you also pair it with an integer, in this case your rank:

#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>

int main(int argc, char **argv) {

    int rank, size;
    const int locn=5;
    int localarr[locn];

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    srand(rank);
    for (int i=0; i<locn; i++) 
        localarr[i] = rand() % 100;

    for (int proc=0; proc<size; proc++) {
        if (rank == proc) {
            printf("Rank %2d has values: ",rank);
            for (int i=0; i<locn; i++)
                printf(" %d ", localarr[i]);
            printf("\n");
        }
        MPI_Barrier(MPI_COMM_WORLD);
    }

    int localres[2];
    int globalres[2];
    localres[0] = localarr[0];
    for (int i=1; i<locn; i++) 
        if (localarr[i] < localres[0]) localres[0] = localarr[i];

    localres[1] = rank;

    MPI_Allreduce(localres, globalres, 1, MPI_2INT, MPI_MINLOC, MPI_COMM_WORLD);

    if (rank == 0) {
        printf("Rank %d has lowest value of %d\n", globalres[1], globalres[0]);
    }

    MPI_Finalize();

    return 0;
}

And running you get:

$ mpirun -np 5 ./minloc
Rank  0 has values:  83  86  77  15  93 
Rank  1 has values:  83  86  77  15  93 
Rank  2 has values:  90  19  88  75  61 
Rank  3 has values:  46  85  68  40  25 
Rank  4 has values:  1  83  74  26  63 
Rank 4 has lowest value of 1

If the value you’re reducing isn’t an integer, (say, a double), you create a structure containing the reduction value and the integer index, and use the appropriate MPI pair data type. (eg, MPI_DOUBLE_INT).

Updated further: Ok, just for fun, doing it with our own reduction operation and our own type to implement two indices:

#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>

typedef struct dbl_twoindex_struct {
    double val;
    int    rank;
    int    posn;
} dbl_twoindex;


void minloc_dbl_twoindex(void *in, void *inout, int *len, MPI_Datatype *type){
    /* ignore type, just trust that it's our dbl_twoindex type */
    dbl_twoindex *invals    = in;
    dbl_twoindex *inoutvals = inout;

    for (int i=0; i<*len; i++) {
        if (invals[i].val < inoutvals[i].val) {
            inoutvals[i].val  = invals[i].val;
            inoutvals[i].rank = invals[i].rank;
            inoutvals[i].posn = invals[i].posn;
        }
    }

    return;
}


int main(int argc, char **argv) {

    int rank, size;
    const int locn=5;
    double localarr[locn];

    dbl_twoindex local, global;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    /* create our new data type */
    MPI_Datatype mpi_dbl_twoindex;
    MPI_Datatype types[3] = { MPI_DOUBLE, MPI_INT, MPI_INT };
    MPI_Aint disps[3] = { offsetof(dbl_twoindex, val),
                     offsetof(dbl_twoindex, rank),
                     offsetof(dbl_twoindex, posn),  };
    int lens[3] = {1,1,1};
    MPI_Type_create_struct(3, lens, disps, types, &mpi_dbl_twoindex);
    MPI_Type_commit(&mpi_dbl_twoindex);

   /* create our operator */
    MPI_Op mpi_minloc_dbl_twoindex;
    MPI_Op_create(minloc_dbl_twoindex, 1, &mpi_minloc_dbl_twoindex);

    srand(rank);
    for (int i=0; i<locn; i++)
        localarr[i] = 1.*rand()/RAND_MAX;

    for (int proc=0; proc<size; proc++) {
        if (rank == proc) {
            printf("Rank %2d has values: ",rank);
            for (int i=0; i<locn; i++)
                printf(" %8.4lf ", localarr[i]);
            printf("\n");
        }
        MPI_Barrier(MPI_COMM_WORLD);
    }

    local.val  = localarr[0];
    local.posn = 0;
    for (int i=1; i<locn; i++)
        if (localarr[i] < local.val) {
                local.val  = localarr[i];
                local.posn = i;
        }
    local.rank = rank;

    MPI_Allreduce(&local, &global, 1, mpi_dbl_twoindex, mpi_minloc_dbl_twoindex, MPI_COMM_WORLD);

    if (rank == 0) {
        printf("Rank %d has lowest value of %8.4lf in position %d.\n", global.rank, global.val, global.posn);
    }

    MPI_Op_free(&mpi_minloc_dbl_twoindex);
    MPI_Type_free(&mpi_dbl_twoindex);
    MPI_Finalize();

    return 0;
}

Running gives

$ mpirun -np 5 ./minloc2
Rank  0 has values:    0.8402    0.3944    0.7831    0.7984    0.9116 
Rank  1 has values:    0.8402    0.3944    0.7831    0.7984    0.9116 
Rank  2 has values:    0.7010    0.8097    0.0888    0.1215    0.3483 
Rank  3 has values:    0.5614    0.2250    0.3931    0.4439    0.2850 
Rank  4 has values:    0.9165    0.1340    0.1912    0.2601    0.2143 
Rank 2 has lowest value of   0.0888 in position 2.