Imported empty headers/parts rules from Rspamd

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.4.1] - 2023-10-26
+
+## Added
+
+### Changed
+
+### Fixed
+- Dockerfile entrypoint script.
+- `bayes_is_balanced` function.
+
 ## [0.4.0] - 2023-10-25
 
 This version introduces some breaking changes in the configuration file. Please read the [UPGRADING.md](UPGRADING.md) file for more information.

crates/smtp/src/scripts/functions/header.rs

@@ -75,6 +75,14 @@ pub fn fn_attachment_name<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec<Variabl
         .into()
 }
 
+pub fn fn_mime_part_len<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec<Variable>) -> Variable {
+    ctx.message()
+        .part(ctx.part())
+        .map(|p| p.len())
+        .unwrap_or_default()
+        .into()
+}
+
 pub fn fn_thread_name<'x>(_: &'x Context<'x, SieveContext>, v: Vec<Variable>) -> Variable {
     v[0].transform(|s| thread_name(s).into())
 }

crates/smtp/src/scripts/functions/mod.rs

@@ -110,6 +110,7 @@ pub fn register_functions() -> FunctionMap<SieveContext> {
         .with_function_no_args("is_body", fn_is_body)
         .with_function_no_args("var_names", fn_is_var_names)
         .with_function_no_args("attachment_name", fn_attachment_name)
+        .with_function_no_args("mime_part_len", fn_mime_part_len)
 }
 
 pub trait ApplyString<'x> {

resources/config/spamfilter/maps/scores.map

@@ -358,3 +358,7 @@ XM_CASE 0.5
 XM_UA_NO_VERSION 0.01
 X_PHP_EVAL 4.0
 ZERO_WIDTH_SPACE_URL 7.0
+SHORT_PART_BAD_HEADERS 7.0
+MISSING_ESSENTIAL_HEADERS 7.0
+SINGLE_SHORT_PART 0.0
+COMPLETELY_EMPTY 7.0

resources/config/spamfilter/scripts/composites.sieve

@@ -1,3 +1,7 @@
+if eval "t.MISSING_ESSENTIAL_HEADERS && t.SINGLE_SHORT_PART" {
+	let "t.SHORT_PART_BAD_HEADERS" "1";
+}
+
 if eval "t.FORGED_RECIPIENTS && t.MAILLIST" {
 	let "t.FORGED_RECIPIENTS_MAILLIST" "1";
 }

resources/config/spamfilter/scripts/headers.sieve

@@ -32,8 +32,11 @@ if eval "header.x-priority.exists" {
 }
 
 let "unique_header_names" "to_lowercase(header.Content-Type:Content-Transfer-Encoding:Date:From:Sender:Reply-To:To:Cc:Bcc:Message-ID:In-Reply-To:References:Subject[*].raw_name)";
-if eval "count(unique_header_names) != count(dedup(unique_header_names))" {
+let "unique_header_names_len" "count(unique_header_names)";
+if eval "unique_header_names_len != count(dedup(unique_header_names))" {
     let "t.MULTIPLE_UNIQUE_HEADERS" "1";
+} elsif eval "unique_header_names_len == 0" {
+    let "t.MISSING_ESSENTIAL_HEADERS" "1";
 }
 
 # Wrong case X-Mailer

resources/config/spamfilter/scripts/mime.sieve

@@ -8,6 +8,8 @@ if eval "!header.mime-version.exists" {
 
 let "has_text_part" "0";
 let "is_encrypted" "0";
+let "parts_num" "0";
+let "parts_max_len" "0";
 
 if eval "header.Content-Type.exists && !header.Content-Disposition:Content-Transfer-Encoding:MIME-Version.exists && !eq_ignore_case(header.Content-Type, 'text/plain')" {
     # Only Content-Type header without other MIME headers
@@ -97,49 +99,60 @@ foreverypart {
         } elsif eval "subtype == 'encrypted'" {
             set "is_encrypted" "1";
         }
-    } elsif eval "type == 'text'" {
+    } else {
-        # MIME text part claims to be ASCII but isn't
+        if eval "type == 'text'" {
-        if eval "cte == '' || cte == '7bit'" {
+            # MIME text part claims to be ASCII but isn't
-            if eval "!is_ascii(part.raw)" {
+            if eval "cte == '' || cte == '7bit'" {
-                let "t.R_BAD_CTE_7BIT" "1";
+                if eval "!is_ascii(part.raw)" {
-            }
+                    let "t.R_BAD_CTE_7BIT" "1";
-        } else {
+                }
-            if eval "cte == 'base64'" {
+            } else {
-                if eval "is_ascii(part.text)" {
+                if eval "cte == 'base64'" {
-                    # Has text part encoded in base64 that does not contain any 8bit characters
+                    if eval "is_ascii(part.text)" {
-                    let "t.MIME_BASE64_TEXT_BOGUS" "1";
+                        # Has text part encoded in base64 that does not contain any 8bit characters
-                } else {
+                        let "t.MIME_BASE64_TEXT_BOGUS" "1";
-                    # Has text part encoded in base64
+                    } else {
-                    let "t.MIME_BASE64_TEXT" "1";
+                        # Has text part encoded in base64
+                        let "t.MIME_BASE64_TEXT" "1";
+                    }
+                }
+
+                if eval "subtype == 'plain' && is_empty(header.content-type.attr.charset)" {
+                    # Charset header is missing
+                    let "t.R_MISSING_CHARSET" "1";
                 }
             }
-
+            let "has_text_part" "1";
-            if eval "subtype == 'plain' && is_empty(header.content-type.attr.charset)" {
+        } elsif eval "type == 'application'" {
-                # Charset header is missing
+            if eval "subtype == 'pkcs7-mime'" {
-                let "t.R_MISSING_CHARSET" "1";
+                let "t.ENCRYPTED_SMIME" "1";
+                let "part_is_attachment" "0";
+            } elsif eval "subtype == 'pkcs7-signature'" {
+                let "t.SIGNED_SMIME" "1";
+                let "part_is_attachment" "0";
+            } elsif eval "subtype == 'pgp-encrypted'" {
+                let "t.ENCRYPTED_PGP" "1";
+                let "part_is_attachment" "0";
+            } elsif eval "subtype == 'pgp-signature'" {
+                let "t.SIGNED_PGP" "1";
+                let "part_is_attachment" "0";
+            } elsif eval "subtype == 'octet-stream'" {
+                if eval "!is_encrypted &&
+                        !header.content-id.exists && 
+                        (!header.content-disposition.exists || 
+                        (!eq_ignore_case(header.content-disposition.type, 'attachment') && 
+                        is_empty(header.content-disposition.attr.filename)))" {
+                    let "t.CTYPE_MISSING_DISPOSITION" "1";
+                }
             }
         }
-        let "has_text_part" "1";
+
-    } elsif eval "type == 'application'" {
+        # Increase part count
-        if eval "subtype == 'pkcs7-mime'" {
+        let "parts_num" "parts_num + 1";
-            let "t.ENCRYPTED_SMIME" "1";
+        if eval "parts_num == 1" {
-            let "part_is_attachment" "0";
+            let "parts_len" "mime_part_len()";
-        } elsif eval "subtype == 'pkcs7-signature'" {
+            if eval "parts_len > parts_max_len" {
-            let "t.SIGNED_SMIME" "1";
+                let "parts_max_len" "parts_len";
-            let "part_is_attachment" "0";
-        } elsif eval "subtype == 'pgp-encrypted'" {
-            let "t.ENCRYPTED_PGP" "1";
-            let "part_is_attachment" "0";
-        } elsif eval "subtype == 'pgp-signature'" {
-            let "t.SIGNED_PGP" "1";
-            let "part_is_attachment" "0";
-        } elsif eval "subtype == 'octet-stream'" {
-            if eval "!is_encrypted &&
-                     !header.content-id.exists && 
-                     (!header.content-disposition.exists || 
-                      (!eq_ignore_case(header.content-disposition.type, 'attachment') && 
-                       is_empty(header.content-disposition.attr.filename)))" {
-                let "t.CTYPE_MISSING_DISPOSITION" "1";
             }
         }
     }
@@ -201,10 +214,18 @@ foreverypart {
 
 }
 
+# Message contains both text and encrypted parts
 if eval "has_text_part && (t.ENCRYPTED_SMIME || t.SIGNED_SMIME || t.ENCRYPTED_PGP || t.SIGNED_PGP)" {
     let "t.BOGUS_ENCRYPTED_AND_TEXT" "1";
 }
 
+# Message contains only one short part
+if eval "parts_num == 1 && parts_max_len < 64" {
+    let "t.SINGLE_SHORT_PART" "1";
+} elsif eval "parts_max_len == 0" {
+    let "t.COMPLETELY_EMPTY" "1";
+}
+
 # Check for mixed script in body
 if eval "!is_single_script(text_body)" {
     let "t.R_MIXED_CHARSET" "1";

tests/resources/smtp/antispam/headers.test

@@ -77,3 +77,9 @@ List-Unsubscribe: 1
 Subject: test
 
 Test
+<!-- NEXT TEST -->
+expect MISSING_ESSENTIAL_HEADERS
+
+X-Other: test
+
+Test

tests/resources/smtp/antispam/mime.test

@@ -1,17 +1,17 @@
-expect MISSING_MIME_VERSION
+expect MISSING_MIME_VERSION SINGLE_SHORT_PART
 
 Content-Type: text/plain; charset="us-ascii"
 
 Test
 <!-- NEXT TEST -->
-expect MV_CASE
+expect MV_CASE SINGLE_SHORT_PART
 
 Content-Type: text/plain; charset="us-ascii"
 Mime-Version: 1.0
 
 Test
 <!-- NEXT TEST -->
-expect CTE_CASE CT_EXTRA_SEMI
+expect CTE_CASE CT_EXTRA_SEMI SINGLE_SHORT_PART
 
 Content-Type: text/plain; charset="us-ascii"; 
 Content-Transfer-Encoding: 7Bit
@@ -19,7 +19,7 @@ MIME-Version: 1.0
 
 Test
 <!-- NEXT TEST -->
-expect BROKEN_CONTENT_TYPE
+expect BROKEN_CONTENT_TYPE SINGLE_SHORT_PART
 
 Content-Type: ; tag=1
 Content-Transfer-Encoding: 7bit
@@ -27,13 +27,13 @@ MIME-Version: 1.0
 
 Test
 <!-- NEXT TEST -->
-expect MIME_HEADER_CTYPE_ONLY MISSING_MIME_VERSION
+expect MIME_HEADER_CTYPE_ONLY MISSING_MIME_VERSION SINGLE_SHORT_PART
 
 Content-Type: text/html; charset="us-ascii"
 
 Test
 <!-- NEXT TEST -->
-expect R_BAD_CTE_7BIT
+expect R_BAD_CTE_7BIT SINGLE_SHORT_PART
 
 Content-Type: text/plain
 Content-Transfer-Encoding: 7bit
@@ -41,7 +41,7 @@ MIME-Version: 1.0
 
 Téstíng
 <!-- NEXT TEST -->
-expect R_MISSING_CHARSET
+expect R_MISSING_CHARSET SINGLE_SHORT_PART
 
 Content-Type: text/plain
 Content-Transfer-Encoding: 8bit
@@ -49,7 +49,7 @@ MIME-Version: 1.0
 
 Test
 <!-- NEXT TEST -->
-expect MIME_BASE64_TEXT_BOGUS
+expect MIME_BASE64_TEXT_BOGUS SINGLE_SHORT_PART
 
 Content-Type: text/plain; charset="utf-8"
 Content-Transfer-Encoding: base64
@@ -58,7 +58,7 @@ MIME-Version: 1.0
 aGVsbG8gd29ybGQK
 
 <!-- NEXT TEST -->
-expect MIME_BASE64_TEXT
+expect MIME_BASE64_TEXT SINGLE_SHORT_PART
 
 Content-Type: text/plain; charset="utf-8"
 Content-Transfer-Encoding: base64
@@ -281,7 +281,7 @@ Content-Transfer-Encoding: 7bit
 </html>
 --boundary--
 <!-- NEXT TEST -->
-expect CTYPE_MISSING_DISPOSITION HAS_ATTACHMENT
+expect CTYPE_MISSING_DISPOSITION HAS_ATTACHMENT SINGLE_SHORT_PART
 
 Content-Type: application/octet-stream
 MIME-Version: 1.0
@@ -326,7 +326,7 @@ this is a test
 
 --boundary--
 <!-- NEXT TEST -->
-expect CTYPE_MISSING_DISPOSITION HAS_ATTACHMENT
+expect CTYPE_MISSING_DISPOSITION HAS_ATTACHMENT SINGLE_SHORT_PART
 
 Content-Type: application/octet-stream
 MIME-Version: 1.0
@@ -353,7 +353,7 @@ this is a test
 
 --boundary--
 <!-- NEXT TEST -->
-expect R_MIXED_CHARSET
+expect R_MIXED_CHARSET SINGLE_SHORT_PART
 
 Content-Type: text/plain; charset="utf-8"
 Content-Transfer-Encoding: 8bit

tests/src/smtp/inbound/antispam.rs

@@ -212,7 +212,12 @@ async fn antispam() {
         )
         .replace("%CFG_PATH%", base_path.as_path().to_str().unwrap());
     let base_path = base_path.join("scripts");
-    let script_config = fs::read_to_string(base_path.join("config.sieve")).unwrap();
+    let script_config = fs::read_to_string(base_path.join("config.sieve"))
+        .unwrap()
+        .replace(
+            "AUTOLEARN_SPAM_HAM_BALANCE\" \"0.9",
+            "AUTOLEARN_SPAM_HAM_BALANCE\" \"0.0",
+        );
     let script_prelude = fs::read_to_string(base_path.join("prelude.sieve")).unwrap();
     let mut all_scripts = script_config.clone() + "\n" + script_prelude.as_str();
     for test_name in tests {