feat(waf)：增加请求日志统计 (#4048)

plugins/openresty/waf/conf/waf.conf

@@ -8,4 +8,5 @@ lua_shared_dict waf_accesstoken 10m;
 lua_package_path "/usr/local/openresty/1pwaf/?.lua;/usr/local/openresty/1pwaf/lib/?.lua;;";
 init_by_lua_file  /usr/local/openresty/1pwaf/init.lua;
 access_by_lua_file /usr/local/openresty/1pwaf/waf.lua;
-log_by_lua_file /usr/local/openresty/1pwaf/log_and_traffic.lua;
+log_by_lua_file /usr/local/openresty/1pwaf/log_and_traffic.lua;
+init_worker_by_lua_file /usr/local/openresty/1pwaf/worker.lua;

plugins/openresty/waf/db.lua

@@ -48,7 +48,7 @@ function _M.init_db()
     local status = {}
     if not check_table("attack_log",wafdb) then
         status = wafdb:exec([[
-            CREATE TABLE attack_log (
+            CREATE TABLE req_logs (
                 id TEXT PRIMARY KEY,
                 ip TEXT,
                 ip_iso TEXT,
@@ -72,7 +72,8 @@ function _M.init_db()
                 nginx_log TEXT,
                 blocking_time INTEGER,
                 action TEXT,
-                is_block INTEGER
+                is_block INTEGER,
+                is_attack INTEGER
             )]])
     end
 
@@ -95,12 +96,13 @@ function _M.init_db()
                 day TEXT,
                 req_count INTEGER,
                 attack_count INTEGER,
-                count_4xx INTEGER,
+                count4xx INTEGER,
-                count_5xx INTEGER
+                count5xx INTEGER,
+                create_date DATETIME
             )]])
         ngx.log(ngx.ERR, "init waf_stat status"..status)
     end
-
+    
     ngx.log(ngx.ERR, "init db success")
 end
 

plugins/openresty/waf/lib/action.lua

@@ -138,8 +138,7 @@ function _M.exec_action(rule_config, match_rule, data)
     ngx.ctx.rule_table = rule_config
     ngx.ctx.action = action
     ngx.ctx.hitData = data
-    
+    ngx.ctx.is_attack = true
-    ngx.ctx.isAttack = true
 
     if rule_config.ipBlock and rule_config.ipBlock == 'on' then
         _M.block_ip(ngx.ctx.ip, rule_config)
@@ -151,13 +150,7 @@ function _M.exec_action(rule_config, match_rule, data)
 
     attack_count(rule_config.type)
 
-    local msg = "访问 IP " .. ngx.ctx.ip .. " 访问 URL" .. ngx.var.uri .. " 触发动作 " .. action .. "User-Agent" .. ngx.ctx.ua .. "  规则类型 " .. rule_config.type .. "  规则 " .. rule_config.rule
+    local msg = "访问 IP " .. ngx.ctx.ip .. " 访问 URL" .. ngx.var.uri .. " 触发动作 " .. action .. " User-Agent " .. ngx.ctx.ua .. "  规则类型 " .. rule_config.type .. "  规则 " .. rule_config.rule
-    if ngx.ctx.country then
-        msg = "国家" .. ngx.ctx.country["zh"]
-    end
-    if ngx.ctx.city then
-        msg = "省份" .. ngx.ctx.city["zh"]
-    end
 
     ngx.log(ngx.ERR, msg)
     if action == "allow" then

plugins/openresty/waf/lib/utils.lua

@@ -6,10 +6,6 @@ local ipairs = ipairs
 local type = type
 local find_str = string.find
 local gmatch_str = string.gmatch
-local gsub_str = string.gsub
-local format_str = string.format
-local random = math.random
-
 
 local _M = {}
 
@@ -150,15 +146,6 @@ function _M.is_intranet_address(ip_addr)
     end
 end
 
-
-function _M.uuid()
-    local template ='xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'
-    return gsub_str(template, '[xy]', function (c)
-        local v = (c == 'x') and random(0, 0xf) or random(8, 0xb)
-        return format_str('%x', v)
-    end)
-end
-
 function _M.get_wafdb(waf_db_path)
     local ok, sqlite3 = pcall(function()
         return require "lsqlite3"

plugins/openresty/waf/log_and_traffic.lua

@@ -8,32 +8,58 @@ local upper_str = string.upper
 local tonumber = tonumber
 local pairs = pairs
 
-
+local function write_req_Log(wafdb,attack)
-local function writeAttackLog()
+    local rule_table = nil
-    local rule_table = ngx.ctx.rule_table
+    local action = ""
-    local action = ngx.ctx.action
+    local rule = nil
-    local rule = rule_table.rule
+    local rule_type = ""
-
+    local is_attack = 0
-    local rule_type = rule_table.type
+    local is_block = 0
-    if not rule_type then
+    local blocking_time = 0
-        rule_type = "default"
+ 
+    if attack then
+        rule_table = ngx.ctx.rule_table
+        action = ngx.ctx.action
+        rule = rule_table.rule
+        rule_type = rule_table.type
+        is_attack = 1
+        if not rule_type then
+            rule_type = "default"
+        end
+        if ngx.ctx.ipBlocked then
+            is_block = 1
+            blocking_time = tonumber(rule_table.ipBlockTime)
+        end
     end
 
     local real_ip = ngx.ctx.ip
     local geoip = ngx.ctx.geoip
-    local country = geoip.country
+    local country
+    local province
+    local longitude = 0.0
+    local latitude = 0.0
+    local iso = "CN"
+    if geoip then
+        country = geoip.country
+        province = geoip.province
+        longitude = geoip.longitude
+        latitude = geoip.latitude
+        iso = geoip.iso
+    else 
+        ngx.log(ngx.ERR, real_ip .. " 无法获取地址")
+    end
     if not country then
-        country["zh"] = "unknown"
+        country = {
-        country["en"] = "unknown"
+            ["zh"] = "unknown",
+            ["en"] = "unknown"
+        }
     end
-    local province = geoip.province
     if not province then
-        province["zh"] = "unknown"
+        province = {
-        province["en"] = "unknown"
+            ["zh"] = "unknown",
+            ["en"] = "unknown"
+        }
     end
-    local longitude = geoip.longitude
-    local latitude = geoip.latitude
-    local iso = geoip.iso
 
     local method = ngx.req.get_method()
     local uri = ngx.var.request_uri
@@ -41,8 +67,6 @@ local function writeAttackLog()
     local host = ngx.var.server_name
     local protocol = ngx.var.server_protocol
     local website_key = ngx.ctx.website_key
- 
-
    
     local logs_str = method .. "  " .. uri .. " "..protocol.."\n"
     local headers = ngx.req.get_headers(20000)
@@ -53,38 +77,24 @@ local function writeAttackLog()
         end
         logs_str = logs_str .. upper_str(k) .. ": " .. value .. "\n"
     end
-    
-
-    local isBlock = 0
-    local blocking_time = 0
-    if ngx.ctx.ipBlocked then
-        isBlock = 1
-        blocking_time = tonumber(rule_table.ipBlockTime)
-    end
 
     local log_id = uuid()
     local time = os.time()
     local localtime = os.date("%Y-%m-%d %H:%M:%S", time)
-
-
-    local wafdb = utils.get_wafdb(config.waf_db_path)
-    if wafdb == nil then
-        return false
-    end
     
     local insertQuery = [[
-        INSERT INTO attack_log (
+        INSERT INTO req_logs (
             id, ip, ip_iso, ip_country_zh, ip_country_en,
             ip_province_zh, ip_province_en, ip_longitude, ip_latitude,
             time, localtime, server_name,  website_key, host, method, 
             uri, user_agent, rule_type,match_rule, match_value,
-            nginx_log, blocking_time, action, is_block
+            nginx_log, blocking_time, action, is_block,is_attack
         ) VALUES (
             :id, :real_ip, :iso,  :country_zh, :country_en,
             :province_zh, :province_en,:longitude, :latitude,
             :time, :localtime, :server_name,:host, :website_key, :method, 
             :uri, :ua, :rule_type, :match_rule, :match_value,
-            :logs_str, :blocking_time, :action, :is_block
+            :logs_str, :blocking_time, :action, :is_block, :is_attack
         )
      ]]
 
@@ -114,7 +124,8 @@ local function writeAttackLog()
         logs_str = logs_str, 
         blocking_time = blocking_time or 0,
         action = action,
-        is_block = isBlock
+        is_block = is_block,
+        is_attack = is_attack
     }
 
     local code = stmt:step()
@@ -161,11 +172,48 @@ local function count_not_found()
     end
 end
 
-if config.is_waf_on() then
+local function count_req_status(wafdb,is_attack)
-    count_not_found()
+    local today = ngx.today()
-    local isAttack = ngx.ctx.isAttack
+    local status = ngx.status
 
-    if isAttack then
+    local stmt_exist = wafdb:prepare("SELECT COUNT(*) FROM waf_stat WHERE day = ?")
-        writeAttackLog()
+    stmt_exist:bind_values(today)
+    stmt_exist:step()
+    local count = stmt_exist:get_uvalues()
+
+    local req_count_update = 1
+    local count_4xx_update = (status >= 400 and status < 500) and 1 or 0
+    local count_5xx_update = (status >= 500) and 1 or 0
+    local attack_count_update = is_attack and 1 or 0
+    local code = 0
+    
+    if count > 0 then
+        local stmt = wafdb:prepare("UPDATE waf_stat SET req_count = req_count + ?, count4xx = count4xx + ?, count5xx = count5xx + ?, attack_count = attack_count + ? WHERE day = ?")
+        stmt:bind_values(req_count_update, count_4xx_update, count_5xx_update, attack_count_update, today)
+        code = stmt:step()
+        stmt:finalize()
+    else
+        local stmt = wafdb:prepare("INSERT INTO waf_stat (day, req_count, count4xx, count5xx, attack_count,create_date) VALUES (?, ?, ?, ?, ?,DATETIME('now'))")
+        stmt:bind_values(today, req_count_update, count_4xx_update, count_5xx_update, attack_count_update)
+        code = stmt:step()
+        stmt:finalize()
+    end
+
+    if code ~= 101 then
+        local errorMsg = wafdb:errmsg()
+        if errorMsg then
+            ngx.log(ngx.ERR, "update waf_stat error ", errorMsg .. "  ")
+        end
+    end
+end
+
+if config.is_waf_on() then
+    count_not_found()
+    local is_attack = ngx.ctx.is_attack
+    
+    local wafdb = utils.get_wafdb(config.waf_db_path)
+    if wafdb ~= nil then
+        count_req_status(wafdb,is_attack)
+        write_req_Log(wafdb,is_attack)
     end
 end

plugins/openresty/waf/waf.lua

@@ -146,7 +146,7 @@ end
 if config.is_waf_on() then
     init()
     waf_api()
-
+    
     if lib.is_white_ip() then
         return true
     end