From cf25895f60a0de265b35b0711779c600bf990d8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dominik=20Madar=C3=A1sz?= <zaklaus@outlook.com>
Date: Fri, 30 Aug 2024 16:01:46 +0200
Subject: [PATCH] fix shadow blending + bugs

---
 demos/09-shadows.c                    |  6 ++-
 engine/art/shaderlib/light.glsl       | 19 +++++++---
 engine/art/shaderlib/model_fs.glsl    |  4 ++
 engine/art/shaderlib/model_vs.glsl    |  1 +
 engine/art/shaderlib/shadowmap.glsl   | 53 +++++++++++----------------
 engine/art/shaderlib/surface.glsl     |  3 +-
 engine/art/shaders/fs_shadow_vsm.glsl |  2 +-
 engine/joint/v4k.h                    | 14 +------
 engine/split/v4k_render.c             |  7 +---
 engine/split/v4k_render.h             |  7 ----
 engine/v4k.c                          |  7 +---
 engine/v4k.h                          |  7 ----
 12 files changed, 52 insertions(+), 78 deletions(-)

diff --git a/demos/09-shadows.c b/demos/09-shadows.c
index 538fdfa..85c3af8 100644
--- a/demos/09-shadows.c
+++ b/demos/09-shadows.c
@@ -33,14 +33,16 @@ int main(int argc, char** argv) {
         lit.type = LIGHT_POINT;
         lit.cast_shadows = true;
         // lit.shadow_distance = 1.0f;
-        lit.falloff.linear = 2.0f;
+        // lit.falloff.linear = 0.5f;
+        lit.falloff.quadratic = 0.1f;
     }
     light_t lit2 = light(); {
         lit2.type = LIGHT_POINT;
         lit2.cast_shadows = true;
         lit2.diffuse = vec3(1, 0.7, 0.8);
         // lit2.shadow_distance = 1.0f;
-        lit2.falloff.linear = 2.0f;
+        // lit2.falloff.linear = 0.5f;
+        lit2.falloff.quadratic = 0.1f;
     }
     light_t lit3 = light(); {
         lit3.type = LIGHT_SPOT;
diff --git a/engine/art/shaderlib/light.glsl b/engine/art/shaderlib/light.glsl
index e919882..3c3b35a 100644
--- a/engine/art/shaderlib/light.glsl
+++ b/engine/art/shaderlib/light.glsl
@@ -1,12 +1,11 @@
 #ifndef LIGHT_GLSL
 #define LIGHT_GLSL
 
-#include "brdf.glsl"
-
-uniform int u_num_lights;
-
+#define MAX_LIGHTS 16
 #define NUM_SHADOW_CASCADES 6
 
+#include "brdf.glsl"
+
 struct light_t {
     int type;
     vec3 diffuse;
@@ -29,13 +28,17 @@ struct light_t {
     mat4 shadow_matrix[NUM_SHADOW_CASCADES];
 };
 
-#define MAX_LIGHTS 16
 const int LIGHT_DIRECTIONAL = 0;
 const int LIGHT_POINT = 1;
 const int LIGHT_SPOT = 2;
 
+uniform int u_num_lights;
 uniform light_t u_lights[MAX_LIGHTS];
 
+#ifdef FS_PASS
+#include "shadowmap.glsl"
+#endif
+
 struct material_t {
     vec3 albedo;
     vec3 normal;
@@ -120,7 +123,11 @@ vec3 lighting(material_t m) {
     vec3 lit = vec3(0,0,0);
 #ifndef SHADING_NONE
     for (int i=0; i<u_num_lights; i++) {
-        lit += shading_light(u_lights[i], m);
+        vec3 lit_contrib = shading_light(u_lights[i], m);
+
+#ifdef FS_PASS
+        lit += lit_contrib * shadowing(i).xyz;
+#endif
     }
 #endif
     return lit;
diff --git a/engine/art/shaderlib/model_fs.glsl b/engine/art/shaderlib/model_fs.glsl
index 25a0cff..7740d2f 100644
--- a/engine/art/shaderlib/model_fs.glsl
+++ b/engine/art/shaderlib/model_fs.glsl
@@ -1,5 +1,6 @@
 #ifndef MODEL_FS_GLSL
 #define MODEL_FS_GLSL
+#define FS_PASS
 
 uniform mat4 model, view, inv_view;
 uniform sampler2D u_texture2d;
@@ -17,6 +18,9 @@ uniform float u_litboost; /// set:1
 
 in vec3 v_position;
 in vec3 v_position_ws;
+in vec4 vpeye;
+in vec4 vneye;
+
 #ifdef RIM
 uniform mat4 M; // RIM
 uniform vec3 u_rimcolor; /// set:0.05,0.05,0.05
diff --git a/engine/art/shaderlib/model_vs.glsl b/engine/art/shaderlib/model_vs.glsl
index 321a86d..c493fe6 100644
--- a/engine/art/shaderlib/model_vs.glsl
+++ b/engine/art/shaderlib/model_vs.glsl
@@ -1,5 +1,6 @@
 #ifndef MODEL_VS_GLSL
 #define MODEL_VS_GLSL
+#define VS_PASS
 
 #ifndef MAX_BONES
 #define MAX_BONES 110
diff --git a/engine/art/shaderlib/shadowmap.glsl b/engine/art/shaderlib/shadowmap.glsl
index a0f5f96..2ee5fe1 100644
--- a/engine/art/shaderlib/shadowmap.glsl
+++ b/engine/art/shaderlib/shadowmap.glsl
@@ -1,17 +1,19 @@
+#ifndef SHADOWMAP_GLSL
+#define SHADOWMAP_GLSL
+
 #include "utils.glsl"
 
-in vec4 vpeye;
-in vec4 vneye;
 uniform bool u_shadow_receiver;
 uniform float u_cascade_distances[MAX_LIGHTS * NUM_SHADOW_CASCADES];
 uniform samplerCube shadowMap[MAX_LIGHTS];
 uniform sampler2D shadowMap2D[MAX_LIGHTS * NUM_SHADOW_CASCADES];
 
 const float bias_modifier[NUM_SHADOW_CASCADES] = float[NUM_SHADOW_CASCADES](0.95, 0.35, 0.20, 0.15, 0.15, 0.15);
+// const float bias_modifier[NUM_SHADOW_CASCADES] = float[NUM_SHADOW_CASCADES](0.95, 0.35, 0.20, 0.15);
 
 //// From http://fabiensanglard.net/shadowmappingVSM/index.php
 float shadow_vsm(float distance, vec3 dir, int light_index) {
-    distance = distance/20;
+    distance = distance/200;
     vec2 moments = texture(shadowMap[light_index], dir).rg;
 
     // If the shadow map is sampled outside of its bounds, return 1.0
@@ -52,9 +54,10 @@ float shadow_pcf(float distance, vec3 lightDir, int light_index) {
         cascade_index = max_cascades_range - 1;
     }
 
+    light_t light = u_lights[light_index];
     int matrix_index = cascade_index - min_cascades_range;
 
-    vec4 fragPosLightSpace = u_lights[light_index].shadow_matrix[matrix_index] * vec4(v_position_ws, 1.0);
+    vec4 fragPosLightSpace = light.shadow_matrix[matrix_index] * vec4(v_position_ws, 1.0);
 
     // Perform perspective divide
     vec3 projCoords = fragPosLightSpace.xyz / fragPosLightSpace.w;
@@ -101,44 +104,32 @@ float shadow_pcf(float distance, vec3 lightDir, int light_index) {
     return 1.0 - shadow;
 }
 
-vec4 shadowmap(in vec4 peye, in vec4 neye) {
+vec4 shadowmap(int idx, in vec4 peye, in vec4 neye) {
     vec3 fragment = vec3(peye);
-
     float shadowFactor = 1.0;
-    float totalWeight = 0.0;
-    for (int i = 0; i < u_num_lights; i++) {
-        light_t light = u_lights[i];
-        float factor = 1.0;
-        float weight = 1.0;
+    light_t light = u_lights[idx];
 
-        if (light.processed_shadows) {
-            if (light.type == LIGHT_DIRECTIONAL) {
-                factor = shadow_pcf(-peye.z, light.dir, i);
-                weight = 0.7;
-            } else if (light.type == LIGHT_POINT || light.type == LIGHT_SPOT) {
-                vec3 light_pos = (view * vec4(light.pos, 1.0)).xyz;
-                vec3 dir = light_pos - fragment;
-                vec4 sc = inv_view * vec4(dir, 0.0);
-                factor = shadow_vsm(length(dir), -sc.xyz, i);
-                weight = 1.0;
-            }
-            shadowFactor *= mix(1.0, factor, weight);
-            totalWeight += weight;
+    if (light.processed_shadows) {
+        if (light.type == LIGHT_DIRECTIONAL) {
+            shadowFactor = shadow_pcf(-peye.z, light.dir, idx);
+        } else if (light.type == LIGHT_POINT || light.type == LIGHT_SPOT) {
+            vec3 light_pos = (view * vec4(light.pos, 1.0)).xyz;
+            vec3 dir = light_pos - fragment;
+            vec4 sc = inv_view * vec4(dir, 0.0);
+            shadowFactor = shadow_vsm(length(dir), -sc.xyz, idx);
         }
     }
 
-    // Normalize the shadow factor based on total weight
-    if (totalWeight > 0.0) {
-        shadowFactor = pow(shadowFactor, 1.0 / totalWeight);
-    }
-
     return vec4(vec3(shadowFactor), 1.0);
 }
 
-vec4 shadowing() {
+vec4 shadowing(int idx) {
     if (u_shadow_receiver) {
-        return shadowmap(vpeye, vneye);
+        return shadowmap(idx, vpeye, vneye);
     } else {
         return vec4(1.0);
     }
 }
+
+#endif
+
diff --git a/engine/art/shaderlib/surface.glsl b/engine/art/shaderlib/surface.glsl
index 2b09569..1f3d197 100644
--- a/engine/art/shaderlib/surface.glsl
+++ b/engine/art/shaderlib/surface.glsl
@@ -4,7 +4,6 @@
 #include "sh_lighting.glsl"
 #include "rimlight.glsl"
 #include "light.glsl"
-#include "shadowmap.glsl"
 
 struct surface_t {
     vec3 normal;
@@ -195,7 +194,7 @@ surface_t surface() {
 
     s.albedo *= v_color;
 
-    s.light_direct *= shadowing().xyz;
+    // s.light_direct *= shadowing().xyz;
 
     s.fragcolor = s.albedo;
     s.fragcolor.rgb *= s.light_direct + s.light_indirect;
diff --git a/engine/art/shaders/fs_shadow_vsm.glsl b/engine/art/shaders/fs_shadow_vsm.glsl
index 4092fa0..1214226 100644
--- a/engine/art/shaders/fs_shadow_vsm.glsl
+++ b/engine/art/shaders/fs_shadow_vsm.glsl
@@ -8,7 +8,7 @@ uniform int shadow_technique;
 
 void main() {
     if (shadow_technique == SHADOW_VSM) {
-        float depth = length(v_position) / 20;
+        float depth = length(v_position) / 200;
 
         float moment1 = depth;
         float moment2 = depth * depth;
diff --git a/engine/joint/v4k.h b/engine/joint/v4k.h
index 2e0f013..59cb8cf 100644
--- a/engine/joint/v4k.h
+++ b/engine/joint/v4k.h
@@ -17310,13 +17310,6 @@ API void    light_update(unsigned num_lights, light_t *lv);
 // -----------------------------------------------------------------------------
 // shadowmaps
 
-// #ifndef VSMCUBE
-// #define VSMCUBE 0
-// #endif
-// #ifndef VSMBLUR
-// #define VSMBLUR 1
-// #endif
-
 typedef struct shadowmap_t {
     mat44 V;
     mat44 PV;
@@ -382149,10 +382142,7 @@ unsigned shader_geom(const char *gs, const char *vs, const char *fs, const char
                 puts("--- gs:");
                 shader_print(gs);
             }
-        }
-        if (status == GL_FALSE) {
             PANIC("ERROR: shader(): Shader/program link: %s\n", buf);
-            return 0;
         }
 
         glDeleteShader(vert);
@@ -383442,8 +383432,8 @@ shadowmap_t shadowmap(int vsm_texture_width, int pcf_texture_width) { // = 512,
     s.blur_scale = 0.5f;
     s.cascade_splits[0] = 0.1f;
     s.cascade_splits[1] = 0.3f;
-    s.cascade_splits[2] = 0.5f;
-    s.cascade_splits[3] = 0.7f;
+    s.cascade_splits[2] = 0.7f;
+    s.cascade_splits[3] = 1.0f;
     s.cascade_splits[4] = 1.0f;
     s.cascade_splits[5] = 1.0f; /* sticks to camera far plane */
 
diff --git a/engine/split/v4k_render.c b/engine/split/v4k_render.c
index b023d17..acafb38 100644
--- a/engine/split/v4k_render.c
+++ b/engine/split/v4k_render.c
@@ -412,10 +412,7 @@ unsigned shader_geom(const char *gs, const char *vs, const char *fs, const char
                 puts("--- gs:");
                 shader_print(gs);
             }
-        }
-        if (status == GL_FALSE) {
             PANIC("ERROR: shader(): Shader/program link: %s\n", buf);
-            return 0;
         }
 
         glDeleteShader(vert);
@@ -1705,8 +1702,8 @@ shadowmap_t shadowmap(int vsm_texture_width, int pcf_texture_width) { // = 512,
     s.blur_scale = 0.5f;
     s.cascade_splits[0] = 0.1f;
     s.cascade_splits[1] = 0.3f;
-    s.cascade_splits[2] = 0.5f;
-    s.cascade_splits[3] = 0.7f;
+    s.cascade_splits[2] = 0.7f;
+    s.cascade_splits[3] = 1.0f;
     s.cascade_splits[4] = 1.0f;
     s.cascade_splits[5] = 1.0f; /* sticks to camera far plane */
 
diff --git a/engine/split/v4k_render.h b/engine/split/v4k_render.h
index 9ce7e55..8265e3f 100644
--- a/engine/split/v4k_render.h
+++ b/engine/split/v4k_render.h
@@ -342,13 +342,6 @@ API void    light_update(unsigned num_lights, light_t *lv);
 // -----------------------------------------------------------------------------
 // shadowmaps
 
-// #ifndef VSMCUBE
-// #define VSMCUBE 0
-// #endif
-// #ifndef VSMBLUR
-// #define VSMBLUR 1
-// #endif
-
 typedef struct shadowmap_t {
     mat44 V;
     mat44 PV;
diff --git a/engine/v4k.c b/engine/v4k.c
index 31016c5..7df4fc4 100644
--- a/engine/v4k.c
+++ b/engine/v4k.c
@@ -17211,10 +17211,7 @@ unsigned shader_geom(const char *gs, const char *vs, const char *fs, const char
                 puts("--- gs:");
                 shader_print(gs);
             }
-        }
-        if (status == GL_FALSE) {
             PANIC("ERROR: shader(): Shader/program link: %s\n", buf);
-            return 0;
         }
 
         glDeleteShader(vert);
@@ -18504,8 +18501,8 @@ shadowmap_t shadowmap(int vsm_texture_width, int pcf_texture_width) { // = 512,
     s.blur_scale = 0.5f;
     s.cascade_splits[0] = 0.1f;
     s.cascade_splits[1] = 0.3f;
-    s.cascade_splits[2] = 0.5f;
-    s.cascade_splits[3] = 0.7f;
+    s.cascade_splits[2] = 0.7f;
+    s.cascade_splits[3] = 1.0f;
     s.cascade_splits[4] = 1.0f;
     s.cascade_splits[5] = 1.0f; /* sticks to camera far plane */
 
diff --git a/engine/v4k.h b/engine/v4k.h
index 9cae95d..027625e 100644
--- a/engine/v4k.h
+++ b/engine/v4k.h
@@ -3377,13 +3377,6 @@ API void    light_update(unsigned num_lights, light_t *lv);
 // -----------------------------------------------------------------------------
 // shadowmaps
 
-// #ifndef VSMCUBE
-// #define VSMCUBE 0
-// #endif
-// #ifndef VSMBLUR
-// #define VSMBLUR 1
-// #endif
-
 typedef struct shadowmap_t {
     mat44 V;
     mat44 PV;