Basic Stencil Shadows with OpenGL

Back to OpenGL Tutorial Index

This OpenGL tutorial will show you how to use the stencil buffer to select a plane to be drawn to, then use the stencil buffer to draw a basic shadow to that plane.

To achieve the effect of shadows that remain on a plane
the best method is to use the stencil buffer. With this
we can set the plane that the shadow is to be drawn on
and any part that is not drawn within the selected plane
is then clipped from the view.

Now the stencil buffer basically just draws the shape
again and changes is accordingly, setting it to the
selected part of the scene.

To do this we have to set up the stencil buffer by changing the
code in the main function to:
glutInitDisplayMode (GLUT_DOUBLE | GLUT_RGBA | GLUT_STENCIL);
notice that we are adding GLUT_STENCIL.

That then leads to another call in the display function
so that just like clearing the depth and color buffers,
we clear the stencil buffer with:
glClearStencil(0);
And change the clear line to:
glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

After that we need to set the plane for the stencil buffer
to draw to.
So we clear the color and depth masks, then we enable the stencil
test with:
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
glDepthMask(GL_FALSE);
glEnable(GL_STENCIL_TEST);

Then we set the stencil function to replace the data in our
selected plane with whatever we choose to. We set it to
replace with:
glStencilFunc(GL_ALWAYS, 1, 0xFFFFFFFF);
glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);

Next we set the plane that we want the stencil buffer to
draw to. I have used my bench function for this with:
bench();
So it will take the coordinates for my bench and use
that as the plane to draw to.

Then we turn on the color mask, the depth mask and
set the stencil function to keep whatever we say next.
So it is replacing what we had before, with what we are
about to add. We do this with:
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
glDepthMask(GL_TRUE);
glStencilFunc(GL_EQUAL, 1, 0xFFFFFFFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);

Now it is time to draw the shadow.
I first disable texturing so our shadow appears plain.
I then disable the depth testing so that the shadow will
appear behind our bench.
I then draw the square flipped upside down, and translated
into the drawing plane.
I then set the rotation as the squares rotation and
color it black.
Then I re-enable the depth testing and the texturing.
This is all done with:
glDisable(GL_TEXTURE_2D);
glDisable(GL_DEPTH_TEST);
glPushMatrix();
glScalef(1.0f, -1.0f, 1.0f);
glTranslatef(0,2,0);
glRotatef(angle,0,1,0);
glColor4f(0,0,0,1);
square();
glPopMatrix();
glEnable(GL_DEPTH_TEST);
glEnable(GL_TEXTURE_2D);

Next I simply disable the stencil test with:
glDisable(GL_STENCIL_TEST);

After that I just draw the scene as usual with
the bench set to an alpha blend so that the
shadow appears transparent.

Now that should give you an idea of how to use the stencil
buffer to create basic shadows.

Keep visiting the site as I am working on
shadows that change shape in accordance with the light
position.

#include <GL/gl.h>
#include <GL/glut.h>
#include <windows.h>
#include <stdio.h>
#include <iostream.h>

float angle = 0;

GLuint texture[40];

void freetexture (GLuint texture) {
glDeleteTextures( 1, &texture );
}

loadtextures (const char *filename, float width, float height) {
GLuint texture;

unsigned char *data;
FILE *file;

file = fopen( filename, "rb" );
if ( file == NULL ) return 0;

data = (unsigned char *)malloc( width * height * 3 );
fread( data, width * height * 3, 1, file );

fclose( file );

glGenTextures( 1, &texture );
glBindTexture( GL_TEXTURE_2D, texture );
glTexEnvf( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE );
glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST );
glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
gluBuild2DMipmaps( GL_TEXTURE_2D, 3, width, height, GL_RGB, GL_UNSIGNED_BYTE, data );

data = NULL;

return texture;
}

void square (void) {
glPushMatrix();
glBindTexture(GL_TEXTURE_2D,texture[0]);
glTranslatef(0,2.5,0);
glScalef(2,2,2);
glBegin(GL_QUADS);
glTexCoord2f(1,0);
glVertex3f(-1,-1,0);
glTexCoord2f(1,1);
glVertex3f(-1,1,0);
glTexCoord2f(0,1);
glVertex3f(1,1,0);
glTexCoord2f(0,0);
glVertex3f(1,-1,0);
glEnd();
glPopMatrix();
}

void bench (void) {
glPushMatrix();
glColor4f(1,1,1,0.7);
glBindTexture(GL_TEXTURE_2D,texture[1]);
glTranslatef(0,-2.5,0);
glScalef(4,2,4);
glBegin(GL_QUADS);
glTexCoord2f(1,0);
glVertex3f(-1,-1,1);
glTexCoord2f(1,1);
glVertex3f(-1,1,-0.5);
glTexCoord2f(0,1);
glVertex3f(1,1,-0.5);
glTexCoord2f(0,0);
glVertex3f(1,-1,1);
glEnd();
glPopMatrix();
}

void display (void) {
glClearStencil(0); //clear the stencil buffer
glClearDepth(1.0f);
glClearColor (1.0,1.0,1.0,1);
glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT); //clear the buffers
glLoadIdentity();

glTranslatef(0, 0, -10);

//start
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE); //disable the color mask
glDepthMask(GL_FALSE); //disable the depth mask

glEnable(GL_STENCIL_TEST); //enable the stencil testing

glStencilFunc(GL_ALWAYS, 1, 0xFFFFFFFF);
glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE); //set the stencil buffer to replace our next lot of data

bench(); //set the data plane to be replaced

glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); //enable the color mask
glDepthMask(GL_TRUE); //enable the depth mask

glStencilFunc(GL_EQUAL, 1, 0xFFFFFFFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP); //set the stencil buffer to keep our next lot of data

glDisable(GL_TEXTURE_2D); //disable texturing of the shadow
glDisable(GL_DEPTH_TEST); //disable depth testing of the shadow
glPushMatrix();
glScalef(1.0f, -1.0f, 1.0f); //flip the shadow vertically
glTranslatef(0,2,0); //translate the shadow onto our drawing plane
glRotatef(angle,0,1,0); //rotate the shadow accordingly
glColor4f(0,0,0,1); //color the shadow black
square(); //draw our square as the shadow
glPopMatrix();
glEnable(GL_DEPTH_TEST); //enable depth testing
glEnable(GL_TEXTURE_2D); //enable texturing

glDisable(GL_STENCIL_TEST); //disable the stencil testing
//end

glEnable(GL_BLEND); //enable alpha blending
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); //set teh alpha blending

bench(); //draw our bench

glDisable(GL_BLEND); //disable alpha blending

glRotatef(angle,0,1,0); //rotate the square
square(); //draw the square

glutSwapBuffers();
angle++;
}

void init (void) {
glEnable(GL_DEPTH_TEST);
glDepthFunc(GL_LEQUAL);
glShadeModel (GL_SMOOTH);
glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST);
glEnable(GL_TEXTURE_2D);

texture[0] = loadtextures("texture.raw", 256,256);
texture[1] = loadtextures("water.raw", 256,256);
}

void reshape (int w, int h) {
glViewport (0, 0, (GLsizei)w, (GLsizei)h);
glMatrixMode (GL_PROJECTION);
glLoadIdentity ();
gluPerspective (60, (GLfloat)w / (GLfloat)h, 0.1, 1000.0);
glMatrixMode (GL_MODELVIEW);
}

int main (int argc, char **argv) {
glutInit (&argc, argv);
glutInitDisplayMode (GLUT_DOUBLE | GLUT_RGBA | GLUT_STENCIL); //add a stencil buffer to the window
glutInitWindowSize (500, 500);
glutInitWindowPosition (100, 100);
glutCreateWindow ("A basic OpenGL Window");
init();
glutDisplayFunc (display);
glutIdleFunc (display);
glutReshapeFunc (reshape);
glutMainLoop ();
return 0;
}

Download C++ Source Code for this Tutorial

Download Visual Basic Source Code for this Tutorial (soon)