Engage!

1 files changed, 14 insertions, 0 deletions

diff --git a/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp b/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp
new file mode 100644
index 0000000..bda9dea
--- /dev/null
+++ b/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp
@@ -0,0 +1,14 @@
+#version 450
+
+#include "glu_head.glsl"
+
+float op(float a, float b) {
+    float xi = min(a, p.limit);
+    float gi = max(min(b, p.limit), -p.limit);
+
+    float out_glu = xi / (1.0f + exp(-xi * p.alpha));
+    out_glu = out_glu * (1.0f + gi);
+    return out_glu;
+}
+
+#include "glu_main.glsl"