lua: expose hash function to lua scripts

Expose md5, sha1, and sha256 to Lua scripts with
`require("suricata.hashing")`.

Ticket: 7073

rust/src/ffi/hashing.rs

@@ -23,6 +23,7 @@ use std::os::raw::c_char;
 
 pub const SC_SHA1_LEN: usize = 20;
 pub const SC_SHA256_LEN: usize = 32;
+pub const SC_MD5_LEN: usize = 16;
 
 // Length of hex digests without trailing NUL.
 pub const SC_MD5_HEX_LEN: usize = 32;
@@ -172,11 +173,17 @@ pub unsafe extern "C" fn SCMd5Free(hasher: &mut SCMd5) {
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn SCMd5HashBuffer(buf: *const u8, buf_len: u32, out: *mut u8, len: u32) {
+pub unsafe extern "C" fn SCMd5HashBuffer(
+    buf: *const u8, buf_len: u32, out: *mut u8, len: u32,
+) -> bool {
+    if len as usize != SC_MD5_LEN {
+        return false;
+    }
     let data = std::slice::from_raw_parts(buf, buf_len as usize);
     let output = std::slice::from_raw_parts_mut(out, len as usize);
     let hash = Md5::new().chain(data).finalize();
     output.copy_from_slice(&hash);
+    true
 }
 
 /// C binding for a function to MD5 hash a single buffer to a hex string.
@@ -230,9 +237,18 @@ mod test {
             SCSha256Update(hasher, bytes.as_ptr(), bytes.len() as u32);
             SCSha256Update(hasher, bytes.as_ptr(), bytes.len() as u32);
             let hex = [0_u8; SC_SHA256_HEX_LEN + 1];
-            SCSha256FinalizeToHex(hasher, hex.as_ptr() as *mut c_char, (SC_SHA256_HEX_LEN + 1) as u32);
-            let string = std::ffi::CStr::from_ptr(hex.as_ptr() as *mut c_char).to_str().unwrap();
-            assert_eq!(string, "22a48051594c1949deed7040850c1f0f8764537f5191be56732d16a54c1d8153");
+            SCSha256FinalizeToHex(
+                hasher,
+                hex.as_ptr() as *mut c_char,
+                (SC_SHA256_HEX_LEN + 1) as u32,
+            );
+            let string = std::ffi::CStr::from_ptr(hex.as_ptr() as *mut c_char)
+                .to_str()
+                .unwrap();
+            assert_eq!(
+                string,
+                "22a48051594c1949deed7040850c1f0f8764537f5191be56732d16a54c1d8153"
+            );
         }
     }
 
@@ -250,10 +266,15 @@ mod test {
             SCMd5Update(hasher, bytes.as_ptr(), bytes.len() as u32);
             SCMd5Update(hasher, bytes.as_ptr(), bytes.len() as u32);
             let hex = [0_u8; SC_MD5_HEX_LEN + 1];
-            SCMd5FinalizeToHex(hasher, hex.as_ptr() as *mut c_char, (SC_MD5_HEX_LEN + 1) as u32);
-            let string = std::ffi::CStr::from_ptr(hex.as_ptr() as *mut c_char).to_str().unwrap();
+            SCMd5FinalizeToHex(
+                hasher,
+                hex.as_ptr() as *mut c_char,
+                (SC_MD5_HEX_LEN + 1) as u32,
+            );
+            let string = std::ffi::CStr::from_ptr(hex.as_ptr() as *mut c_char)
+                .to_str()
+                .unwrap();
             assert_eq!(string, "5216ddcc58e8dade5256075e77f642da");
         }
     }
-
 }

src/Makefile.am

@@ -512,6 +512,7 @@ noinst_HEADERS = \
 	util-lua-dnp3-objects.h \
 	util-lua-dns.h \
 	util-lua.h \
+	util-lua-hashlib.h \
 	util-lua-hassh.h \
 	util-lua-http.h \
 	util-lua-ja3.h \
@@ -1060,6 +1061,7 @@ libsuricata_c_a_SOURCES = \
 	util-lua-dnp3.c \
 	util-lua-dnp3-objects.c \
 	util-lua-dns.c \
+	util-lua-hashlib.c \
 	util-lua-hassh.c \
 	util-lua-http.c \
 	util-lua-ja3.c \

src/util-lua-hashlib.c

@@ -0,0 +1,292 @@
+/* Copyright (C) 2025 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * Hashing library for Lua.
+ *
+ * Usage:
+ *
+ * local hashing = require("suricata.hashing")
+ *
+ * -- One shot hash
+ * hash = hashing.sha256_digtest("www.suricata.io")
+ *
+ * -- Incremental hashing
+ * hasher = hashing.sha256()
+ * hasher:update("www.")
+ * hasher:update("suricata.io")
+ * hash = hasher:finalize()
+ *
+ * Support hashes:
+ *
+ * - sha256: sha256(), sha256_digest()
+ * - sha1: sha1(), sha1_digest()
+ * - md5: md5(), md5_digest()
+ */
+
+#include "util-lua-hashlib.h"
+
+#include "lauxlib.h"
+
+#define SHA256_MT "suricata:hashlib:sha256"
+#define SHA1_MT   "suricata:hashlib:sha1"
+#define MD5_MT    "suricata:hashlib:md5"
+
+static int LuaHashLibSha256New(lua_State *L)
+{
+    struct SCSha256 **hasher = lua_newuserdata(L, sizeof(struct SCSha256 *));
+    *hasher = SCSha256New();
+    luaL_getmetatable(L, SHA256_MT);
+    lua_setmetatable(L, -2);
+    return 1;
+}
+
+static int LuaHashLibSha256Update(lua_State *L)
+{
+    struct SCSha256 **hasher = luaL_checkudata(L, 1, SHA256_MT);
+    size_t data_len;
+    const char *data = luaL_checklstring(L, 2, &data_len);
+    SCSha256Update(*hasher, (const uint8_t *)data, data_len);
+    return 0;
+}
+
+static int LuaHashLibSha256Finalize(lua_State *L)
+{
+    struct SCSha256 **hasher = luaL_checkudata(L, 1, SHA256_MT);
+    uint8_t hash[SC_SHA256_LEN];
+
+    SCSha256Finalize(*hasher, hash, sizeof(hash));
+    lua_pushlstring(L, (const char *)hash, sizeof(hash));
+
+    // Finalize consumes the hasher, so set to NULL so its not free'd
+    // during garbage collection.
+    *hasher = NULL;
+
+    return 1;
+}
+
+static int LuaHashLibSha256Hash(lua_State *L)
+{
+    size_t buf_len;
+    const char *input = luaL_checklstring(L, 1, &buf_len);
+
+    size_t out_len = SC_SHA256_LEN;
+    uint8_t output[out_len];
+    bool result = SCSha256HashBuffer((const uint8_t *)input, (uint32_t)buf_len, output, out_len);
+
+    if (!result) {
+        return luaL_error(L, "sha256 hashing failed");
+    }
+
+    lua_pushlstring(L, (const char *)output, out_len);
+
+    return 1;
+}
+
+static int LuaHashLibSha256Gc(lua_State *L)
+{
+    struct SCSha256 **hasher = luaL_checkudata(L, 1, SHA256_MT);
+    if (*hasher) {
+        SCSha256Free(*hasher);
+    }
+    return 0;
+}
+
+static int LuaHashLibSha1New(lua_State *L)
+{
+    struct SCSha1 **hasher = lua_newuserdata(L, sizeof(struct SCSha1 *));
+    *hasher = SCSha1New();
+    luaL_getmetatable(L, SHA1_MT);
+    lua_setmetatable(L, -2);
+    return 1;
+}
+
+static int LuaHashLibSha1Update(lua_State *L)
+{
+    struct SCSha1 **hasher = luaL_checkudata(L, 1, SHA1_MT);
+    size_t data_len;
+    const char *data = luaL_checklstring(L, 2, &data_len);
+    SCSha1Update(*hasher, (const uint8_t *)data, data_len);
+    return 0;
+}
+
+static int LuaHashLibSha1Finalize(lua_State *L)
+{
+    struct SCSha1 **hasher = luaL_checkudata(L, 1, SHA1_MT);
+    uint8_t hash[SC_SHA1_LEN];
+
+    SCSha1Finalize(*hasher, hash, sizeof(hash));
+    lua_pushlstring(L, (const char *)hash, sizeof(hash));
+
+    // Finalize consumes the hasher, so set to NULL so its not free'd
+    // during garbage collection.
+    *hasher = NULL;
+
+    return 1;
+}
+
+static int LuaHashLibSha1Hash(lua_State *L)
+{
+    size_t buf_len;
+    const char *input = luaL_checklstring(L, 1, &buf_len);
+
+    size_t out_len = SC_SHA1_LEN;
+    uint8_t output[out_len];
+    bool result = SCSha1HashBuffer((const uint8_t *)input, (uint32_t)buf_len, output, out_len);
+
+    if (!result) {
+        return luaL_error(L, "sha1 hashing failed");
+    }
+
+    lua_pushlstring(L, (const char *)output, out_len);
+
+    return 1;
+}
+
+static int LuaHashLibSha1Gc(lua_State *L)
+{
+    struct SCSha1 **hasher = luaL_checkudata(L, 1, SHA1_MT);
+    if (*hasher) {
+        SCSha1Free(*hasher);
+    }
+    return 0;
+}
+
+static int LuaHashLibMd5New(lua_State *L)
+{
+    struct SCMd5 **hasher = lua_newuserdata(L, sizeof(struct SCMd5 *));
+    *hasher = SCMd5New();
+    luaL_getmetatable(L, MD5_MT);
+    lua_setmetatable(L, -2);
+    return 1;
+}
+
+static int LuaHashLibMd5Update(lua_State *L)
+{
+    struct SCMd5 **hasher = luaL_checkudata(L, 1, MD5_MT);
+    size_t data_len;
+    const char *data = luaL_checklstring(L, 2, &data_len);
+    SCMd5Update(*hasher, (const uint8_t *)data, data_len);
+    return 0;
+}
+
+static int LuaHashLibMd5Finalize(lua_State *L)
+{
+    struct SCMd5 **hasher = luaL_checkudata(L, 1, MD5_MT);
+    uint8_t hash[SC_MD5_LEN];
+
+    SCMd5Finalize(*hasher, hash, sizeof(hash));
+    lua_pushlstring(L, (const char *)hash, sizeof(hash));
+
+    // Finalize consumes the hasher, so set to NULL so its not free'd
+    // during garbage collection.
+    *hasher = NULL;
+
+    return 1;
+}
+
+static int LuaHashLibMd5Digest(lua_State *L)
+{
+    size_t buf_len;
+    const char *input = luaL_checklstring(L, 1, &buf_len);
+
+    size_t out_len = SC_MD5_LEN;
+    uint8_t output[out_len];
+    bool result = SCMd5HashBuffer((const uint8_t *)input, (uint32_t)buf_len, output, out_len);
+
+    if (!result) {
+        lua_pushnil(L);
+        lua_pushstring(L, "md5 failed");
+        return 2;
+    }
+
+    lua_pushlstring(L, (const char *)output, out_len);
+
+    return 1;
+}
+
+static int LuaHashLibMd5Gc(lua_State *L)
+{
+    struct SCMd5 **hasher = luaL_checkudata(L, 1, MD5_MT);
+    if (*hasher) {
+        SCMd5Free(*hasher);
+    }
+    return 0;
+}
+
+static const struct luaL_Reg hashlib[] = {
+    // clang-format off
+    { "sha256_digest", LuaHashLibSha256Hash },
+    { "sha256", LuaHashLibSha256New },
+    { "sha1_digest", LuaHashLibSha1Hash },
+    { "sha1", LuaHashLibSha1New },
+    { "md5_digest", LuaHashLibMd5Digest },
+    { "md5", LuaHashLibMd5New },
+    { NULL, NULL },
+    // clang-format on
+};
+
+static const struct luaL_Reg sha256_meta[] = {
+    // clang-format off
+    { "update", LuaHashLibSha256Update },
+    { "finalize", LuaHashLibSha256Finalize },
+    { "__gc", LuaHashLibSha256Gc },
+    { NULL, NULL },
+    // clang-format on
+};
+
+static const struct luaL_Reg sha1_meta[] = {
+    // clang-format off
+    { "update", LuaHashLibSha1Update },
+    { "finalize", LuaHashLibSha1Finalize },
+    { "__gc", LuaHashLibSha1Gc },
+    { NULL, NULL },
+    // clang-format on
+};
+
+static const struct luaL_Reg md5_meta[] = {
+    // clang-format off
+    { "update", LuaHashLibMd5Update },
+    { "finalize", LuaHashLibMd5Finalize },
+    { "__gc", LuaHashLibMd5Gc },
+    { NULL, NULL },
+    // clang-format on
+};
+
+int SCLuaLoadHashlib(lua_State *L)
+{
+    luaL_newmetatable(L, SHA256_MT);
+    lua_pushvalue(L, -1);
+    lua_setfield(L, -2, "__index");
+    luaL_setfuncs(L, sha256_meta, 0);
+
+    luaL_newmetatable(L, SHA1_MT);
+    lua_pushvalue(L, -1);
+    lua_setfield(L, -2, "__index");
+    luaL_setfuncs(L, sha1_meta, 0);
+
+    luaL_newmetatable(L, MD5_MT);
+    lua_pushvalue(L, -1);
+    lua_setfield(L, -2, "__index");
+    luaL_setfuncs(L, md5_meta, 0);
+
+    luaL_newlib(L, hashlib);
+
+    return 1;
+}