Question

In order to use normal mapping in GLSL shaders, you need to know the normal, tangent and bitangent vectors of each vertex. RenderMonkey makes this easy by providing it’s own predefined variables (rm_tangent and rm_binormal) for this. I am trying to add this functionality to my own 3d engine. Apparently it is possible to calculate the tangent and bi tangent of each vertex in a triangle using each vertex’s xyz coordinates, uv texture coordinates and normal vector. After some searching I devised this function to calculate the tangent and bitangent for each vertex in my triangle structure.

void CalculateTangentSpace(void) {
    float x1 = m_vertices[1]->m_pos->Get(0) - m_vertices[0]->m_pos->Get(0);
    float x2 = m_vertices[2]->m_pos->Get(0) - m_vertices[0]->m_pos->Get(0);
    float y1 = m_vertices[1]->m_pos->Get(1) - m_vertices[0]->m_pos->Get(1);
    float y2 = m_vertices[2]->m_pos->Get(1) - m_vertices[0]->m_pos->Get(1);
    float z1 = m_vertices[1]->m_pos->Get(2) - m_vertices[0]->m_pos->Get(2);
    float z2 = m_vertices[2]->m_pos->Get(2) - m_vertices[0]->m_pos->Get(2);

    float u1 = m_vertices[1]->m_texCoords->Get(0) - m_vertices[0]->m_texCoords->Get(0);
    float u2 = m_vertices[2]->m_texCoords->Get(0) - m_vertices[0]->m_texCoords->Get(0);
    float v1 = m_vertices[1]->m_texCoords->Get(1) - m_vertices[0]->m_texCoords->Get(1);
    float v2 = m_vertices[2]->m_texCoords->Get(1) - m_vertices[0]->m_texCoords->Get(1);

    float r = 1.0f/(u1 * v2 - u2 * v1);

    Vec3<float> udir((v2 * x1 - v1 * x2) * r, (v2 * y1 - v1 * y2) * r, (v2 * z1 - v1 * z2) * r);
    Vec3<float> vdir((u1 * x2 - u2 * x1) * r, (u1 * y2 - u2 * y1) * r, (u1 * z2 - u2 * z1) * r);

    Vec3<float> tangent[3];
    Vec3<float> tempNormal;

    tempNormal = *m_vertices[0]->m_normal;
    tangent[0]=(udir-tempNormal*(Vec3Dot(tempNormal, udir)));
    m_vertices[0]->m_tangent=&(tangent[0].Normalize());
    m_vertices[0]->m_bitangent=Vec3Cross(m_vertices[0]->m_normal, m_vertices[0]->m_tangent);

    tempNormal = *m_vertices[1]->m_normal;
    tangent[1]=(udir-tempNormal*(Vec3Dot(tempNormal, udir)));
    m_vertices[1]->m_tangent=&(tangent[1].Normalize());
    m_vertices[1]->m_bitangent=Vec3Cross(m_vertices[1]->m_normal, m_vertices[1]->m_tangent);

    tempNormal = *m_vertices[2]->m_normal;
    tangent[2]=(udir-tempNormal*(Vec3Dot(tempNormal, udir)));
    m_vertices[2]->m_tangent=&(tangent[2].Normalize());
    m_vertices[2]->m_bitangent=Vec3Cross(m_vertices[2]->m_normal, m_vertices[2]->m_tangent);
}

When I use this function and send the calculated values to my shader, the models look almost like they do in RenderMonkey but they flicker in a very strange way. I traced the problem to the tangent and bitangent I am sending OpenGL. This leads me to suspect that my code is doing something wrong. Can anyone see any problems or have any suggestions for other methods to try?

I should also point out that the above code is very hacky and I have very little understanding of the math behind what is going on.

Answer 1

Found the solution. Much simpler (but still a little hacky) code:

void CalculateTangentSpace(void) {
    float x1 = m_vertices[1]->m_pos->Get(0) - m_vertices[0]->m_pos->Get(0);
    float y1 = m_vertices[1]->m_pos->Get(1) - m_vertices[0]->m_pos->Get(1);
    float z1 = m_vertices[1]->m_pos->Get(2) - m_vertices[0]->m_pos->Get(2);

    float u1 = m_vertices[1]->m_texCoords->Get(0) - m_vertices[0]->m_texCoords->Get(0);

    Vec3<float> tangent(x1/u1, y1/u1, z1/u1);
    tangent = tangent.Normalize();

    m_vertices[0]->m_tangent = new Vec3<float>(tangent);
    m_vertices[1]->m_tangent = new Vec3<float>(tangent);
    m_vertices[2]->m_tangent = new Vec3<float>(tangent);

    m_vertices[0]->m_bitangent=new Vec3<float>(Vec3Cross(m_vertices[0]->m_normal, m_vertices[0]->m_tangent)->Normalize());
    m_vertices[1]->m_bitangent=new Vec3<float>(Vec3Cross(m_vertices[1]->m_normal, m_vertices[1]->m_tangent)->Normalize());
    m_vertices[2]->m_bitangent=new Vec3<float>(Vec3Cross(m_vertices[2]->m_normal, m_vertices[2]->m_tangent)->Normalize());
}