Question

I have a square area on which I have to determine where the mouse pointing.
With D3DXIntersectTri I can tell IF the mouse pointing on it, but I have trouble calculating the x,y,z coordinates.

The drawing from vertex buffer, which initialized with the vertices array:

vertices[0].position = D3DXVECTOR3(-10, 0,  -10);
vertices[1].position = D3DXVECTOR3(-10, 0,   10);
vertices[2].position = D3DXVECTOR3( 10, 0,  -10);
vertices[3].position = D3DXVECTOR3( 10, 0,  -10);
vertices[4].position = D3DXVECTOR3(-10, 0,   10);
vertices[5].position = D3DXVECTOR3( 10, 0,   10);

I have this method so far, this is not giving me the right coordinates (works only on a small part of the area, near two of the edges and more less accurate inside):

BOOL Area::getcoord( Ray& ray, D3DXVECTOR3& coord)
{
    D3DXVECTOR3 rayOrigin, rayDirection;
    rayDirection = ray.direction;
    rayOrigin = ray.origin;

    float d;

    D3DXMATRIX matInverse;
    D3DXMatrixInverse(&matInverse, NULL, &matWorld);

    // Transform ray origin and direction by inv matrix
    D3DXVECTOR3 rayObjOrigin,rayObjDirection;

    D3DXVec3TransformCoord(&rayOrigin, &rayOrigin, &matInverse);
    D3DXVec3TransformNormal(&rayDirection, &rayDirection, &matInverse);
    D3DXVec3Normalize(&rayDirection,&rayDirection);

    float u, v;
    BOOL isHit1, isHit2;

    D3DXVECTOR3 p1, p2, p3;
    p1 = vertices[3].position;
    p2 = vertices[4].position;
    p3 = vertices[5].position;

    isHit1 = D3DXIntersectTri(&p1, &p2, &p3, &rayOrigin, &rayDirection, &u, &v, &d);
    isHit2 = FALSE;

    if(!isHit1)
    {
        p1 = vertices[0].position;
        p2 = vertices[1].position;
        p3 = vertices[2].position;
        isHit2 = D3DXIntersectTri(&p1, &p2, &p3, &rayOrigin, &rayDirection, &u, &v, &d);
    }

    if(isHit1) 
    {
        coord.x = 1 * ((1-u-v)*p3.x + u*p3.y + v*p3.z);
        coord.y = 0.2f;
        coord.z = -1 * ((1-u-v)*p1.x + u*p1.y + v*p1.z);
        D3DXVec3TransformCoord(&coord, &coord, &matInverse);
    }

    if(isHit2) 
    {
        coord.x = -1 * ((1-u-v)*p3.x + u*p3.y + v*p3.z);
        coord.y = 0.2f;
        coord.z = 1 * ((1-u-v)*p1.x + u*p1.y + v*p1.z);
        D3DXVec3TransformCoord(&coord, &coord, &matWorld);
    }
    return isHit1 || isHit2;
}

Answer 1

Barycentric coordinates don’t work the way you used them. u and v define the weight of the source vectors. So if you want to calculate the hit point, you will have to compute

coord = u * p1 + v * p2 + (1 - u - v) * p3

Alternatively you can use the d ray parameter:

coord = rayOrigin + d * rDirection

Both ways should result in the same coordinate.