Question

How can I get this shader to have a smooth edge on the spot light instead of a hard one? In addition, the shader has to cope with a variable value of GL_SPOT_CUTOFF. Note that not all the lights are spot lights — GL_LIGHT0 is a point light.

varying vec3 N;
varying vec3 v;
#define MAX_LIGHTS 2 
void main (void)  
{  
    vec4 finalColour;
    float spotEffect;

    for (int i=0; i<MAX_LIGHTS; i++)
    {
        vec3 L = normalize(gl_LightSource[i].position.xyz - v);   
        vec3 E = normalize(-v);
        vec3 R = normalize(-reflect(L,N));

        spotEffect = dot(normalize(gl_LightSource[i].spotDirection), 
                         normalize(-L));
        if (spotEffect > gl_LightSource[i].spotCosCutoff) {
            vec4 Iamb = gl_FrontLightProduct[i].ambient;    
            vec4 Idiff = gl_FrontLightProduct[i].diffuse * max(dot(N,L), 0.0);
            Idiff = clamp(Idiff, 0.0, 1.0);     
            vec4 Ispec = gl_FrontLightProduct[i].specular 
                         * pow(max(dot(R,E),0.0),0.3*gl_FrontMaterial.shininess);
            Ispec = clamp(Ispec, 0.0, 1.0);
            finalColour += Iamb + Idiff + Ispec;
        }
    }
    gl_FragColor = gl_FrontLightModelProduct.sceneColor + finalColour;

}

The scene looks like this:

Answer 1

This shader from http://www.ozone3d.net/tutorials/glsl_lighting_phong_p3.php produces the soft edges to the spotlight you are after.

[Pixel_Shader]

varying vec3 normal, lightDir, eyeVec;

const float cos_outer_cone_angle = 0.8; // 36 degrees

void main (void)
{
    vec4 final_color =
    (gl_FrontLightModelProduct.sceneColor * gl_FrontMaterial.ambient) +
    (gl_LightSource[0].ambient * gl_FrontMaterial.ambient);

    vec3 L = normalize(lightDir);
    vec3 D = normalize(gl_LightSource[0].spotDirection);

    float cos_cur_angle = dot(-L, D);

    float cos_inner_cone_angle = gl_LightSource[0].spotCosCutoff;

    float cos_inner_minus_outer_angle = 
          cos_inner_cone_angle - cos_outer_cone_angle;

    //****************************************************
    // Don't need dynamic branching at all, precompute 
    // falloff(i will call it spot)
    float spot = 0.0;
    spot = clamp((cos_cur_angle - cos_outer_cone_angle) / 
           cos_inner_minus_outer_angle, 0.0, 1.0);
    //****************************************************

    vec3 N = normalize(normal);

    float lambertTerm = max( dot(N,L), 0.0);
    if(lambertTerm > 0.0)
    {
        final_color += gl_LightSource[0].diffuse *
            gl_FrontMaterial.diffuse *
            lambertTerm * spot;

        vec3 E = normalize(eyeVec);
        vec3 R = reflect(-L, N);

        float specular = pow( max(dot(R, E), 0.0),
            gl_FrontMaterial.shininess );

        final_color += gl_LightSource[0].specular *
            gl_FrontMaterial.specular *
            specular * spot;
    }
    gl_FragColor = final_color;