Question

I’ve got a program that uses a few CUDA kernels and this one is taking 50-100ms to run while the others take 0-5ms. I expect this has something to do with all the branching, but I’m not really sure how to reduce it. I’m compiling for a compute capability 2.1 device. If someone could point me in the right direction that would be great.

// chosen using occupancy spreadsheet
#define SCORE_THREADS_PER_BLOCK 448

__device__ double ScoringMatrixVal(double *scoring_matrix, size_t pitch, unsigned int row, unsigned int column) {
  return *((double*)((char*) scoring_matrix + row * pitch) + column);
}

__global__ void ScoreBindingSites(char *input_sequence, unsigned long is_length, unsigned int *rvd_sequence, unsigned int rs_len, double cutoff, unsigned int rvd_num, double *scoring_matrix, size_t sm_pitch, unsigned char *results) {

  int block_seq_index = SCORE_THREADS_PER_BLOCK * (blockIdx.y * gridDim.x + blockIdx.x);
  int thread_id = (blockDim.x * threadIdx.y) + threadIdx.x;
  int seq_index = block_seq_index + thread_id;

  if (seq_index < 1 || seq_index >= is_length || seq_index + rs_len >= is_length - 1) return;

  if (input_sequence[seq_index - 1] == 'T' || input_sequence[seq_index - 1] == 't') {

    double thread_result = 0;

    for (int i = 0; i < rs_len; i++) {

      int rvd_index = i;

      int sm_col = 4;

      char base = input_sequence[seq_index + i];

      if (base == 'A' || base == 'a')    
        sm_col = 0;
      if (base == 'C' || base == 'c')
        sm_col = 1;
      if (base == 'G' || base == 'g')
        sm_col = 2;
      if (base == 'T' || base == 't')
        sm_col = 3;

      thread_result += ScoringMatrixVal(scoring_matrix, sm_pitch, rvd_sequence[rvd_index], sm_col);

    }

    results[seq_index] |= (thread_result < cutoff ? 1UL : 0UL) << (2 * rvd_num);

  } 

  if (input_sequence[seq_index + rs_len] == 'A' || input_sequence[seq_index + rs_len] == 'a') {

    double thread_result = 0;

    for (int i = 0; i < rs_len; i++) {

      int rvd_index = rs_len - i - 1;

      int sm_col = 4;

      char base = input_sequence[seq_index + i];

      if (base == 'A' || base == 'a')    
        sm_col = 3;
      if (base == 'C' || base == 'c')
        sm_col = 2;
      if (base == 'G' || base == 'g')
        sm_col = 1;
      if (base == 'T' || base == 't')
        sm_col = 0;

      thread_result += ScoringMatrixVal(scoring_matrix, sm_pitch, rvd_sequence[rvd_index], sm_col);

    }

    results[seq_index] |= (thread_result < cutoff ? 1UL : 0UL) << (2 * rvd_num + 1);

  }

}

ScoreBindingSites is launched with (32, 14) threads per block and enough blocks to cover the input sequence. The full source can be found here if that would be helpful.

Answer 1

There are a few things you can do to improve this code:

• As has been suggested above, merge the two loops for 'T' and 'A'. This is probably your greatest source of branch divergence as the small cascade of if-statements inside the loop will very probably be compiled as predicated instructions (see Section 5.4.2 of the NVidia CUDA C Programming guide).

• Byte-sized global memory access is a terrible idea. Instead, I would suggest declaring input_sequence, results and base as char4 and, in each iteration of your main loop, do your thing for each value of base.x, base.y, base.z and base.w.

• You may also want to have a closer look at what ScoringMatrixVal is doing. Is it just reading values from memory? If so, could you replace it with constant memory? Or a texture?

Update

As requested, here is what I meant with the second point. I haven’t tested the code, though, so feel free to keep any bugs or typos you find. Note that I’ve assumed, for simplicity, that rs_len is a multiple of four.

// chosen using occupancy spreadsheet
#define SCORE_THREADS_PER_BLOCK 448

__device__ double ScoringMatrixVal(double *scoring_matrix, size_t pitch, unsigned int row, unsigned int column) {
  return scoring_matrix[ row*pitch/sizeof(double) + column ];
}

__global__ void ScoreBindingSites(char4 *input_sequence, unsigned long is_length, unsigned int *rvd_sequence, unsigned int rs_len, double cutoff, unsigned int rvd_num, double *scoring_matrix, size_t sm_pitch, unsigned char *results) {

  int block_seq_index = SCORE_THREADS_PER_BLOCK * (blockIdx.y * gridDim.x + blockIdx.x);
  int thread_id = (blockDim.x * threadIdx.y) + threadIdx.x;
  int seq_index = block_seq_index + thread_id;

  if (seq_index < 1 || seq_index >= is_length || seq_index + rs_len >= is_length - 1) return;

  if (input_sequence[seq_index - 1] == 'T' || input_sequence[seq_index - 1] == 't') {

    double4 thread_result = make_double4( 0 );

    for (int i = 0; i < rs_len/4; i++) {

      int rvd_index = 4*i;

      int4 sm_col = make_int4( 4 );

      char4 base = input_sequence[seq_index + i];

      if (base.x == 'A' || base.x == 'a')    
        sm_col.x = 0;
      else if (base.x == 'C' || base.x == 'c')
        sm_col.x = 1;
      else if (base.x == 'G' || base.x == 'g')
        sm_col.x = 2;
      else if (base.x == 'T' || base.x == 't')
        sm_col.x = 3;
      thread_result.x += ScoringMatrixVal(scoring_matrix, sm_pitch, rvd_sequence[rvd_index + 0], sm_col.x);

      if (base.y == 'A' || base.y == 'a')    
        sm_col.y = 0;
      else if (base.y == 'C' || base.y == 'c')
        sm_col.y = 1;
      else if (base.y == 'G' || base.y == 'g')
        sm_col.y = 2;
      else if (base.y == 'T' || base.y == 't')
        sm_col.y = 3;
      thread_result.y += ScoringMatrixVal(scoring_matrix, sm_pitch, rvd_sequence[rvd_index + 1], sm_col.y);

      if (base.z == 'A' || base.z == 'a')    
        sm_col.z = 0;
      else if (base.z == 'C' || base.z == 'c')
        sm_col.z = 1;
      else if (base.z == 'G' || base.z == 'g')
        sm_col.z = 2;
      else if (base.z == 'T' || base.z == 't')
        sm_col.z = 3;
      thread_result.z += ScoringMatrixVal(scoring_matrix, sm_pitch, rvd_sequence[rvd_index + 2], sm_col.z);

      if (base.w == 'A' || base.w == 'a')    
        sm_col.w = 0;
      else if (base.w == 'C' || base.w == 'c')
        sm_col.w = 1;
      else if (base.w == 'G' || base.w == 'g')
        sm_col.w = 2;
      else if (base.w == 'T' || base.w == 't')
        sm_col.w = 3;
      thread_result.w += ScoringMatrixVal(scoring_matrix, sm_pitch, rvd_sequence[rvd_index + 3], sm_col.w);

    }

    double acc_thread_result = thread_result.x + thread_result.y + thead_result.z + thread_result.w;

    results[seq_index] |= (acc_thread_result < cutoff ? 1UL : 0UL) << (2 * rvd_num);

  }

  if (input_sequence[seq_index + rs_len] == 'A' || input_sequence[seq_index + rs_len] == 'a') {

    ...

  }

}

A few notes:

• I’ve re-written, hopefully correctly, your function ScoringMatrixVal to use regular array access, as the whole mess with pointer arithmetic may be throwing the compiler off.
• I’ve converted your if-statements to a cascade of if-elseif-statements, as they seem mutually exclusive. I’m guessing the compiler will use predicated instructions and will interleave the four if-elseif blocks.
• You may consider replacing all this with a char[256] where everything is set to 4 except for the character codes at A, a, C, c, etc…
• If you convert the if-elseif-statements to a table lookup, you could use two different tables for input_sequence[seq_index - 1] == 'T' and input_sequence[seq_index + rs_len] == 'A', thus keeping it all in one loop.

I hope I haven’t messed-up the code too much and that this helps!