Question

currently im drawing a background, then a transparent triangle into the stencil buffer, then a blue square where the triangle isnt being drawn. i was hoping the effect would be a blue square with a trianglular hole exposing the background. but all im getting is a black triangle drawn over everything else. i think im getting the desired effect on the square but the stencil is also being applied to the background.

my question is how can i adjust the following code to allow the background to be displayed through a triangular hole in the square?

static void Draw(void)
{
glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

//big blue square
glColor3ub(0,255,255);
glBegin(GL_POLYGON);
glVertex3i(5, 5, 0);
glVertex3i(-5, 5, 0);
glVertex3i(-5, -5, 0);
glVertex3i(5, -5, 0);
glEnd();

glStencilFunc(GL_ALWAYS, 1, 1);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);

/* transparent triangle */
glColor4ub(0.0f, 0.0f, 0.0f, 0.0f);
glBegin(GL_POLYGON);
glVertex3i(-4, -4, 0);
glVertex3i(4, -4, 0);
glVertex3i(0, 4, 0);
glEnd();

glStencilFunc(GL_EQUAL, 1, 1);
glStencilOp(GL_INCR, GL_KEEP, GL_DECR);

glStencilFunc(GL_EQUAL, 2, 1);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);

/* blue square */
glColor3ub(0, 0, 200);
glBegin(GL_POLYGON);
glVertex3i(3, 3, 0);
glVertex3i(-3, 3, 0);
glVertex3i(-3, -3, 0);
glVertex3i(3, -3, 0);
glEnd();

if (doubleBuffer) {
    glutSwapBuffers();
} else {
    glFlush();
}
}

Answer 1

Problem one: When drawing the background either disable stencil test before, or switch the stencil function to GL_ALWAYS so that the fragments pass (I recomment turning it off). Then you’re setting two different stencil modi afterwards, but neither will do, what you intend. So here’s the change:

glDisable(GL_STENCIL_TEST);
glStencilMask(0x0);

draw_background();

glEnable(GL_STENCIL_TEST);
glStencilMask(0x1);
glStencilFunc(GL_ALWAYS, 0x1, 0x1);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
glDepthMask(GL_FALSE);

draw_trangle();

glStencilFunc(GL_NOTEQUAL, 0x1, 0x1);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
glDepthMask(GL_TRUE);

draw_blue_square();