Question

my basic question is – what is the proper way to use ‘gluUnProject’ to get a Ray or normal vector from my mouse/cursor position.

I have converted the ‘gluUnProject’ code to C# from the mesa glu C code.

public Vector3 UnProject(Matrix4 projectionMatrix, Matrix4 viewMatrix, int viewX, int viewY, int viewWidth, int viewHeight)
{
    var finalMatrix = viewMatrix.Multiply(projectionMatrix).Invert();

    var vec = new Vector4(this, 1);
    vec.X = (vec.X - viewX) / viewWidth;
    vec.Y = (vec.Y - viewY) / viewHeight;
    vec = (vec * 2) - 1;
    vec.Y = -vec.Y;

    var outVec = vec.Transform(finalMatrix);
    outVec.X /= outVec.W;
    outVec.Y /= outVec.W;
    outVec.Z /= outVec.W;

    return outVec.ToVector3();
}

I then call this method from my Camera object:

public Vector3 UnProject(Vector3 screenPosition)
{
    return screenPosition.UnProject(ProjectionMatrix, ViewMatrix, ViewPort.Location.X, ViewPort.Location.Y, ViewPort.Size.X, ViewPort.Size.Y);
}

To get my normal vector from the ray I do:

var rayLoc = camera.UnProject(new Vector3(Game.Mouse.Location, 0));
var rayLoc2 = camera.UnProject(new Vector3(Game.Mouse.Location, 1));
var rayDirection = (rayLoc2 - rayLoc).Normalize();

Everything works fine until I move my LookAt point over say 1 in the X and Rotate the camera a little on the X and Y axis. After that the vector is no longer correct. I don’t know what i’m doing wrong??

EDIT: After a closer look I think my Matrix4.Invert method may be the cause. I’m trying to use Row major, any ideas?

public Matrix4 Invert()
{
    var mat = new Matrix4(
        new Vector4(
            Y.Y*Z.Z*W.W - Y.Y*Z.W*W.Z - Z.Y*Y.Z*W.W + Z.Y*Y.W*W.Z + W.Y*Y.Z*Z.W - W.Y*Y.W*Z.Z,
            -Y.X*Z.Z*W.W + Y.X*Z.W*W.Z + Z.X*Y.Z*W.W - Z.X*Y.W*W.Z - W.X*Y.Z*Z.W + W.X*Y.W*Z.Z,
            Y.X*Z.Y*W.W - Y.X*Z.W*W.Y - Z.X*Y.Y*W.W + Z.X*Y.Y*W.Y + W.X*Y.Y*Z.W - W.X*Y.W*Z.Y,
            -Y.X*Z.Y*W.Z + Y.X*Z.Z*W.Y + Z.X*Y.Y*W.Z - Z.X*Y.Z*W.Y - W.X*Y.Y*Z.Z + W.X*Y.Z*W.Y),
        new Vector4(
            -X.Y*Z.Z*W.W + X.Y*Z.W*W.Z + Z.Y*X.Z*W.W - Z.Y*X.W*W.Z - W.Y*X.Z*Z.W + W.Y*X.W*Z.Z,
            X.X*Z.Z*W.W - X.X*Z.W*W.Z - Z.X*X.Z*W.W + Z.X*X.W*W.Z + W.X*X.Z*Z.W - W.X*X.W*Z.Z,
            -X.X*Z.Y*W.W + X.X*Z.W*W.Y + Z.X*X.Y*W.W - Z.X*X.W*W.Y - W.X*X.Y*Z.W + W.X*X.W*Z.Y,
            X.X*Z.Y*W.Z - X.X*Z.Z*W.Y - Z.X*X.Y*W.Z + Z.X*X.Z*W.Y + W.X*X.Y*Z.Z - W.X*X.Z*Z.Y),
        new Vector4(
            X.Y*Y.Z*W.W - X.Y*Y.W*W.Z - Y.Y*X.Z*W.W + Y.Y*X.W*W.Z + W.Y*X.Z*Y.W - W.Y*X.W*Y.Z,
            -X.X*Y.Z*W.W + X.X*Y.W*W.Z + Y.X*X.Z*W.W - Y.X*X.W*W.Z - W.X*X.Z*Y.W + W.X*X.W*Y.Z,
            X.X*Y.Y*W.W - X.X*Y.W*W.Y - Y.X*X.Y*W.W + Y.X*X.W*W.Y + W.X*X.Y*Y.W - W.X*X.W*Y.Y,
            -X.X*Y.Y*W.Z + X.X*Y.Z*W.Y + Y.X*X.Y*W.Z - Y.X*X.Z*W.Y - W.X*X.Y*Y.Z + W.X*X.Z*Y.Y),
        new Vector4(
            -X.Y*Y.Z*Z.W + X.Y*Y.W*Z.Z + Y.Y*X.Z*Z.W - Y.Y*X.W*Z.Z - Z.Y*X.Z*Y.W + Z.Y*X.W*Y.Z,
            X.X*Y.Z*Z.W - X.X*Y.W*Z.Z - Y.X*X.Z*Z.W + Y.X*X.W*Z.Z + Z.X*X.Z*Y.W - Z.X*X.Z*Y.Z,
            -X.X*Y.Y*Z.W + X.X*Y.W*Z.Y + Y.X*X.Y*Z.W - Y.X*X.W*Z.Y - Z.X*X.Y*Y.W + Z.X*X.W*Y.Y,
            X.X*Y.Y*Z.Z - X.X*Y.Z*Z.Y - Y.X*X.Y*Z.Z + Y.X*X.Z*Z.Y + Z.X*X.Y*Y.Z - Z.X*X.Z*Y.Y));

    Num delta = X.X*mat.X.X + X.Y*mat.Y.X + X.Z*mat.Z.X + X.W*mat.W.X;
    delta = 1 / delta;
    return mat * delta;
}

Answer 1

It turns out my Matrix4.Invert method was wrong. I used the OpenTK for reference in making a Row-Major Invert method.

EDIT: Actually ‘https://github.com/Dav1dde/gl3n’ had a better way of going about it.