Question

I’d like to be able to compare the performance of running some C code from R (using package inline and Rcpp). I am using rbenchmark to do this in R. A trivial example follows:-

In R I have been using:-

library(inline)
## function to calculate a mean:-
mean_fun <- cxxfunction(signature(a = "numeric"), plugin = "Rcpp", body = '
  Rcpp::NumericVector xa(a);
    int n = xa.size();
    double sum = 0;
    for(int i = 0; i < n; i++) {
        sum += xa[i];
    }
  double mean = sum / n;
    return Rcpp::wrap(mean);    
')
x <- rnorm(100000)
require(rbenchmark)
print(benchmark(mean_fun(x), mean(x), 
      columns = c("test", "replications", "elapsed", "relative"),
      replications = 100))

This gives the output:-

         test replications elapsed relative
2 mean_fun(x)          100   0.019    1.000
1     mean(x)          100   0.039    2.053

And in C I have been essentially using the same function:-

#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <time.h>

#define CLOCKTYPE CLOCK_MONOTONIC
#define MAX_ROWS 1000010


struct Double_array_struct {
    int size_array;
    double double_array[MAX_ROWS];
};

struct Double_array_struct *Load_double_array_struct()
{
    /* loads some_numbers.txt and stores the data into an
    Double_array_struct, resizes the memory allocation, 
    returns the pointer to the struct */
    struct Double_array_struct *data = malloc(MAX_ROWS * sizeof(double) +
        sizeof(int));
    data->size_array = MAX_ROWS;
    double num = 0;
    int array_length = 0;

    FILE *myfile = fopen("some_numbers.txt", "r");
    if (myfile == NULL) {
        perror("error opening file");
    } else {
        printf("File successfully opened\n");
        while(fscanf(myfile, "%lf", &num) > 0)
        {
            /*printf("Number = %lf\n", num);*/
            data->double_array[array_length] = num;
            array_length++;
        }
        data->size_array = array_length;
    }
    fclose(myfile);
    printf("array_length: %d\n", array_length);

    printf("re-sizing the data\n");
    data = realloc(data, (array_length * sizeof(double) + sizeof(int)));
    return(data);
}

void Destroy_double_array_struct(struct Double_array_struct *data) {
    /* function to free the memory used for Int array struct */
    assert(data != NULL);
    free(data);
}

double Mean_double_array_struct(struct Double_array_struct *data) {
    /* function to calculate the mean of an array of 
    doubles, when passed to function in the usual structure.
    Returns a double */
    int i;
    double sum = 0; 
    for(i = 0; i < data->size_array; i++){
        sum += data->double_array[i];
    }
    double mean = sum / data->size_array;
    return(mean);
}


int main()
{

    struct Double_array_struct *double_file_data = Load_double_array_struct();

    /* some timings */    
    printf("about to do timings");
    struct timespec tsi, tsf;
    clock_gettime(CLOCKTYPE, &tsi);
    int z;
    int iterations = 100; /*100 iterations to match rbenchmark */
    double new_array_mean[iterations];
    for(z = 0; z < iterations; z++){
        new_array_mean[z] = Mean_double_array_struct(double_file_data);
        /* I've allocated the result to an array to stop the
        compiler complaining that we're never using the result
        of the function */ 
    }
    clock_gettime(CLOCKTYPE, &tsf);
    double elaps_s = difftime(tsf.tv_sec, tsi.tv_sec);
    long elaps_ns = tsf.tv_nsec - tsi.tv_nsec;
    double time_taken = elaps_s + ((double)elaps_ns) / 1.0e9;

    printf("time taken %lf\n", time_taken);
    Destroy_double_array_struct(double_file_data);

    return 0;
}

The file some_numbers.txt is just a file of 100,000 random numbers generated with rnorm(100000) in R.

This gives the output:-

File successfully opened
array_length: 100000
re-sizing the data
about to do timings:
mean: 0.003486
time taken 0.095793

So, is it meaningful to compare the elapsed time from rbenchmark, and the time which I got from the C function? And if so, why does the same function called from R with Rcpp and inline apparently perform better than when it is called from C?

Answer 1

In order to compare things, they have to be comparable. And, generally speaking, a standalone main() implementation is not comparable to R hosting something because of totally different frameworks from the memory allocation on down to modifying other behavior as needed by R (and comparable scripting languages).

You could compare different algorithms both called by R, or as C standalone programs.

You could compare different languages both called by R, or as C standalone programs.

But you cannot really compare the way you did.