|
@@ -0,0 +1,192 @@
|
|
|
+/*
|
|
|
+ * Module for bloom filters
|
|
|
+ *
|
|
|
+ * Philip Gladstone, N1DQ
|
|
|
+ */
|
|
|
+
|
|
|
+#include "module.h"
|
|
|
+#include "lauxlib.h"
|
|
|
+#include "c_types.h"
|
|
|
+#include "../crypto/sha2.h"
|
|
|
+
|
|
|
+#if defined(LUA_USE_MODULES_BLOOM) && !defined(SHA2_ENABLE)
|
|
|
+#error Must have SHA2_ENABLE set for BLOOM module
|
|
|
+#endif
|
|
|
+
|
|
|
+typedef struct {
|
|
|
+ uint8 fns;
|
|
|
+ uint16 size;
|
|
|
+ uint32 occupancy;
|
|
|
+ uint32 buf[];
|
|
|
+} bloom_t;
|
|
|
+
|
|
|
+static bool add_or_check(const uint8 *buf, size_t len, bloom_t *filter, bool add) {
|
|
|
+ SHA256_CTX ctx;
|
|
|
+ SHA256_Init(&ctx);
|
|
|
+ SHA256_Update(&ctx, buf, len);
|
|
|
+
|
|
|
+ char hash[32];
|
|
|
+ SHA256_Final(hash, &ctx);
|
|
|
+
|
|
|
+ int i;
|
|
|
+ uint32 bits = filter->size << 5;
|
|
|
+ uint8 *h = hash;
|
|
|
+ bool prev = true;
|
|
|
+ int hstep = filter->fns > 10 ? 2 : 3;
|
|
|
+ for (i = 0; i < filter->fns; i++) {
|
|
|
+ uint32 val = (((h[0] << 8) + h[1]) << 8) + h[2];
|
|
|
+ h += hstep;
|
|
|
+ val = val % bits;
|
|
|
+
|
|
|
+ uint32 offset = val >> 5;
|
|
|
+ uint32 bit = 1 << (val & 31);
|
|
|
+
|
|
|
+ if (!(filter->buf[offset] & bit)) {
|
|
|
+ prev = false;
|
|
|
+ if (add) {
|
|
|
+ filter->buf[offset] |= bit;
|
|
|
+ filter->occupancy++;
|
|
|
+ } else {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return prev;
|
|
|
+}
|
|
|
+
|
|
|
+static int bloom_filter_check(lua_State *L) {
|
|
|
+ bloom_t *filter = (bloom_t *)luaL_checkudata(L, 1, "bloom.filter");
|
|
|
+ size_t length;
|
|
|
+ const uint8 *buffer = (uint8 *) luaL_checklstring(L, 2, &length);
|
|
|
+
|
|
|
+ bool rc = add_or_check(buffer, length, filter, false);
|
|
|
+
|
|
|
+ lua_pushboolean(L, rc);
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static int bloom_filter_add(lua_State *L) {
|
|
|
+ bloom_t *filter = (bloom_t *)luaL_checkudata(L, 1, "bloom.filter");
|
|
|
+ size_t length;
|
|
|
+ const uint8 *buffer = (uint8 *) luaL_checklstring(L, 2, &length);
|
|
|
+
|
|
|
+ bool rc = add_or_check(buffer, length, filter, true);
|
|
|
+
|
|
|
+ lua_pushboolean(L, rc);
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static int bloom_filter_reset(lua_State *L) {
|
|
|
+ bloom_t *filter = (bloom_t *)luaL_checkudata(L, 1, "bloom.filter");
|
|
|
+
|
|
|
+ memset(filter->buf, 0, filter->size << 2);
|
|
|
+ filter->occupancy = 0;
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static int bloom_filter_info(lua_State *L) {
|
|
|
+ bloom_t *filter = (bloom_t *)luaL_checkudata(L, 1, "bloom.filter");
|
|
|
+
|
|
|
+ lua_pushinteger(L, filter->size << 5);
|
|
|
+ lua_pushinteger(L, filter->fns);
|
|
|
+ lua_pushinteger(L, filter->occupancy);
|
|
|
+
|
|
|
+ // Now calculate the chance that a FP will be returned
|
|
|
+ uint64 prob = 1000000;
|
|
|
+ if (filter->occupancy > 0) {
|
|
|
+ unsigned int ratio = (filter->size << 5) / filter->occupancy;
|
|
|
+ int i;
|
|
|
+
|
|
|
+ prob = ratio;
|
|
|
+
|
|
|
+ for (i = 1; i < filter->fns && prob < 1000000; i++) {
|
|
|
+ prob = prob * ratio;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (prob < 1000000) {
|
|
|
+ // try again with some scaling
|
|
|
+ unsigned int ratio256 = (filter->size << 13) / filter->occupancy;
|
|
|
+
|
|
|
+ uint64 prob256 = ratio256;
|
|
|
+
|
|
|
+ for (i = 1; i < filter->fns && prob256 < 256000000; i++) {
|
|
|
+ prob256 = (prob256 * ratio256) >> 8;
|
|
|
+ }
|
|
|
+
|
|
|
+ prob = prob256 >> 8;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ lua_pushinteger(L, prob > 1000000 ? 1000000 : (int) prob);
|
|
|
+
|
|
|
+ return 4;
|
|
|
+}
|
|
|
+
|
|
|
+static int bloom_create(lua_State *L) {
|
|
|
+ int items = luaL_checkinteger(L, 1);
|
|
|
+ int error = luaL_checkinteger(L, 2);
|
|
|
+
|
|
|
+ int n = error;
|
|
|
+ int logp = 0;
|
|
|
+ while (n > 0) {
|
|
|
+ n = n >> 1;
|
|
|
+ logp--;
|
|
|
+ }
|
|
|
+
|
|
|
+ int bits = -items * logp;
|
|
|
+ bits += bits >> 1;
|
|
|
+
|
|
|
+ bits = (bits + 31) & ~31;
|
|
|
+
|
|
|
+ if (bits < 256) {
|
|
|
+ bits = 256;
|
|
|
+ }
|
|
|
+
|
|
|
+ int size = bits >> 3;
|
|
|
+
|
|
|
+ int fns = bits / items;
|
|
|
+ fns = (fns >> 1) + fns / 6;
|
|
|
+
|
|
|
+ if (fns < 2) {
|
|
|
+ fns = 2;
|
|
|
+ }
|
|
|
+ if (fns > 15) {
|
|
|
+ fns = 15;
|
|
|
+ }
|
|
|
+
|
|
|
+ bloom_t *filter = (bloom_t *) lua_newuserdata(L, sizeof(bloom_t) + size);
|
|
|
+ //
|
|
|
+ // Associate its metatable
|
|
|
+ luaL_getmetatable(L, "bloom.filter");
|
|
|
+ lua_setmetatable(L, -2);
|
|
|
+
|
|
|
+ memset(filter, 0, sizeof(bloom_t) + size);
|
|
|
+ filter->size = size >> 2;
|
|
|
+ filter->fns = fns;
|
|
|
+
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static const LUA_REG_TYPE bloom_filter_map[] = {
|
|
|
+ { LSTRKEY( "add" ), LFUNCVAL( bloom_filter_add ) },
|
|
|
+ { LSTRKEY( "check" ), LFUNCVAL( bloom_filter_check ) },
|
|
|
+ { LSTRKEY( "reset" ), LFUNCVAL( bloom_filter_reset ) },
|
|
|
+ { LSTRKEY( "info" ), LFUNCVAL( bloom_filter_info ) },
|
|
|
+ { LSTRKEY( "__index" ), LROVAL( bloom_filter_map ) },
|
|
|
+ { LNILKEY, LNILVAL }
|
|
|
+};
|
|
|
+
|
|
|
+// Module function map
|
|
|
+static const LUA_REG_TYPE bloom_map[] = {
|
|
|
+ { LSTRKEY( "create" ), LFUNCVAL( bloom_create ) },
|
|
|
+ { LNILKEY, LNILVAL }
|
|
|
+};
|
|
|
+
|
|
|
+LUALIB_API int bloom_open(lua_State *L) {
|
|
|
+ luaL_rometatable(L, "bloom.filter", (void *)bloom_filter_map);
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+NODEMCU_MODULE(BLOOM, "bloom", bloom_map, bloom_open);
|