Question

I am having problems copying data from my device back to the host. My data are arranged in a struct:

typedef struct Array2D {
    double* arr;        
    int rows;       
    int cols;       
} Array2D;

arr is a ‘flat’ array. rows and cols describes the dimensions.

The code below shows how I am trying to copy the data back to the host:

h_output = (Array2D*) malloc(sizeof(Array2D));
cudaMemcpy(h_output, d_output, sizeof(Array2D), cudaMemcpyDeviceToHost);
double* h_arr = (double*) malloc(h_output->cols*h_output->rows*sizeof(double));
cudaMemcpy(h_arr, h_output->arr, h_output->cols*h_output->rows*sizeof(double), cudaMemcpyDeviceToHost);
h_output->arr = h_arr;

However, in the fourth line the execution fails with cuda error 11 (invalid argument). I cannot see why this is happening. The size of the array is correct, and I can access both h_output and h_array from the host and both have ‘real’ addresses.

EDIT
Sorry for the late response to the request for more information (= more code).

I have tested that the pointer d_output->arr is a device pointer, by trying to access the value of the device pointer on the host. As expected, I was not allowed to do that leaving me with the thought that d_output->arr is in fact a valid device pointer.

The code’s objective is to solve Thiele’s differential equation using the fourth order Runge-Kutta method.

class CalculationSpecification
{

    /* FUNCTIONS OMITTED */

public:
    __device__ void RK4_n(CalculationSpecification* cs, CalcData data, Array2D* d_output)
    {
        double* rk4data = (double*)malloc((data.pdata->endYear - data.pdata->startYear + 1)*data.pdata->states*sizeof(double));

        /* CALCULATION STUFF HAPPENS HERE */

        // We know that rows = 51, cols = 1 and that rk4data contains 51 values as it should.
        // This was confirmed by using printf directly in this function.
        d_output->arr = rk4data;
        d_output->rows = data.pdata->endYear - data.pdata->startYear + 1;
        d_output->cols = data.pdata->states;
    }
};


class PureEndowment : CalculationSpecification
{
    /* FUNCTIONS OMITTED */

public:
    __device__ void Compute(Array2D *result, CalcData data)
    {
        RK4_n(this, data, result);
    }
};


__global__ void kernel2(Array2D *d_output)
{
    /* Other code that initializes 'cd'. */
    PureEndowment pe;
    pe.Compute(d_output,cd);
}


void prepareOutputSet(Array2D* h_output, Array2D* d_output, int count)
{
    h_output = (Array2D*) malloc(sizeof(Array2D));
    cudaMemcpy(h_output, d_output, sizeof(Array2D), cudaMemcpyDeviceToHost); // After this call I can read the correct values of row, col as well as the address of the pointer.
    double* h_arr = (double*) malloc(h_output->cols*h_output->rows*sizeof(double));
    cudaMemcpy(h_arr, h_output->arr, h_output->cols*h_output->rows*sizeof(double), cudaMemcpyDeviceToHost)
    h_output->arr = h_arr;
}

int main()
{
    Array2D *h_output, *d_output;
    cudaMalloc((void**)&d_output, sizeof(Array2D));

    kernel2<<<1,1>>>(d_output);
    cudaDeviceSynchronize();

    prepareOutputSet(h_output, d_output, 1);

    getchar();
    return 0;
}

EDIT2

Additionally, I have now tested that the value of d_output->arr when running on the device is identical to the value of h_output->arr after the first cudaMemcpy-call in prepareOutputSet.

Answer 1

This (copying device-allocated memory using cudaMemcpy) is a known limitation in CUDA 4.1. A fix is in the works and will be released in a future version of the CUDA runtime.