expose demux error msgs (#2349)

Author: doomedraven

conf/default/web.conf.default

@@ -78,8 +78,10 @@ enabled = no
 
 #enable linux fields on webgui
 [linux]
-#For advanced users only, can be buggy, linux analysis is work in progress for fun
+# For advanced users only, can be buggy, linux analysis is work in progress for fun
 enabled = no
+# independent of enabled or not. To not show linux options, but process statically those files
+static_only = no
 
 [malscore]
 enabled = no

lib/cuckoo/common/demux.py

@@ -32,7 +32,7 @@
 cuckoo_conf = Config()
 web_cfg = Config("web")
 tmp_path = cuckoo_conf.cuckoo.get("tmppath", "/tmp")
-linux_enabled = web_cfg.linux.get("enabled", False)
+linux_enabled = web_cfg.linux.get("enabled", False) or web_cfg.linux.get("static_only", False)
 
 demux_extensions_list = {
     b".accdr",
@@ -162,7 +162,8 @@ def is_valid_package(package: str) -> bool:
     return any(ptype in package for ptype in VALID_PACKAGES)
 
 
-def _sf_children(child: sfFile) -> bytes:
+# ToDo fix return type
+def _sf_children(child: sfFile):  # -> bytes:
     path_to_extract = ""
     _, ext = os.path.splitext(child.filename)
     ext = ext.lower()
@@ -184,15 +185,17 @@ def _sf_children(child: sfFile) -> bytes:
                 _ = path_write_file(path_to_extract, child.contents)
         except Exception as e:
             log.error(e, exc_info=True)
-    return path_to_extract.encode()
+    return (path_to_extract.encode(), child.platform, child.get_type(), child.get_size())
 
 
-def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) -> List[bytes]:
+# ToDo fix typing need to add str as error msg
+def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True):  # -> List[bytes]:
     retlist = []
     # do not extract from .bin (downloaded from us)
     if os.path.splitext(filename)[1] == b".bin":
-        return retlist
+        return retlist, ""
 
+    # ToDo need to introduce error msgs here
     try:
         password = options2passwd(options) or "infected"
         try:
@@ -201,9 +204,13 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) ->
             unpacked = unpack(filename, check_shellcode=check_shellcode)
 
         if unpacked.package in whitelist_extensions:
-            return [filename]
+            file = File(filename)
+            magic_type = file.get_type()
+            platform = file.get_platform()
+            file_size = file.get_size()
+            return [filename, platform, magic_type, file_size], ""
         if unpacked.package in blacklist_extensions:
-            return [filename]
+            return [], "blacklisted package"
         for sf_child in unpacked.children:
             if sf_child.to_dict().get("children"):
                 retlist.extend(_sf_children(ch) for ch in sf_child.children)
@@ -214,7 +221,7 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) ->
                 retlist.append(_sf_children(sf_child))
     except Exception as e:
         log.error(e, exc_info=True)
-    return list(filter(None, retlist))
+    return list(filter(None, retlist)), ""
 
 
 def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = True, platform: str = ""):  # -> tuple[bytes, str]:
@@ -227,21 +234,29 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
     if isinstance(filename, str) and use_sflock:
         filename = filename.encode()
 
+    error_list = []
     retlist = []
     # if a package was specified, trim if allowed and required
     if package:
-
         if package in ("msix",):
             retlist.append((filename, "windows"))
         else:
             if File(filename).get_size() <= web_cfg.general.max_sample_size or (
                 web_cfg.general.allow_ignore_size and "ignore_size_check" in options
             ):
-                retlist.append((filename, platform))
+                retlist.append((filename, platform, ""))
             else:
                 if web_cfg.general.enable_trim and trim_file(filename):
                     retlist.append((trimmed_path(filename), platform))
-        return retlist
+                else:
+                    error_list.append(
+                        {
+                            os.path.basename(
+                                filename
+                            ): "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option"
+                        }
+                    )
+        return retlist, error_list
 
     # handle quarantine files
     tmp_path = unquarantine(filename)
@@ -259,9 +274,16 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
         if use_sflock:
             if HAS_SFLOCK:
                 retlist = demux_office(filename, password, platform)
-                return retlist
+                return retlist, error_list
             else:
                 log.error("Detected password protected office file, but no sflock is installed: poetry install")
+                error_list.append(
+                    {
+                        os.path.basename(
+                            filename
+                        ): "Detected password protected office file, but no sflock is installed or correct password provided"
+                    }
+                )
 
     # don't try to extract from Java archives or executables
     if (
@@ -279,7 +301,14 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
         else:
             if web_cfg.general.enable_trim and trim_file(filename):
                 retlist.append((trimmed_path(filename), platform))
-        return retlist
+            else:
+                error_list.append(
+                    {
+                        os.path.basename(filename),
+                        "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option",
+                    }
+                )
+        return retlist, error_list
 
     new_retlist = []
 
@@ -288,26 +317,34 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
         check_shellcode = False
 
     # all in one unarchiver
-    retlist = demux_sflock(filename, options, check_shellcode) if HAS_SFLOCK and use_sflock else []
+    retlist, error_msg = demux_sflock(filename, options, check_shellcode) if HAS_SFLOCK and use_sflock else []
     # if it isn't a ZIP or an email, or we aren't able to obtain anything interesting from either, then just submit the
     # original file
     if not retlist:
+        if error_msg:
+            error_list.append({os.path.basename(filename), error_msg})
         new_retlist.append((filename, platform))
     else:
-        for filename in retlist:
+        for filename, platform, magic_type, file_size in retlist:
             # verify not Windows binaries here:
-            file = File(filename)
-            magic_type = file.get_type()
-            platform = file.get_platform()
             if platform == "linux" and not linux_enabled and "Python" not in magic_type:
+                error_list.append({os.path.basename(filename): "Linux processing is disabled"})
                 continue
 
-            if file.get_size() > web_cfg.general.max_sample_size and not (
+            if file_size > web_cfg.general.max_sample_size and not (
                 web_cfg.general.allow_ignore_size and "ignore_size_check" in options
             ):
                 if web_cfg.general.enable_trim:
                     # maybe identify here
                     if trim_file(filename):
                         filename = trimmed_path(filename)
+            else:
+                error_list.append(
+                    {
+                        os.path.basename(filename),
+                        "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option",
+                    }
+                )
             new_retlist.append((filename, platform))
-    return new_retlist[:10]
+
+    return new_retlist[:10], error_list

lib/cuckoo/common/integrations/file_extra_info.py

@@ -820,6 +820,9 @@ def SevenZip_unpack(file: str, *, filetype: str, data_dictionary: dict, options:
     ):
         return
 
+    if all([pattern in file_data for pattern in (b"AndroidManifest.xml", b"classes.dex")]):
+        return
+
     password = ""
     # Only for real 7zip, breaks others
     password = options.get("password", "infected")

lib/cuckoo/common/web_utils.py

@@ -766,7 +766,7 @@ def download_file(**kwargs):
     if not onesuccess:
         return "error", {"error": f"Provided hash not found on {kwargs['service']}"}
 
-    return "ok", kwargs["task_ids"]
+    return "ok", kwargs["task_ids"], extra_details.get("erros", [])
 
 
 def save_script_to_storage(task_ids, kwargs):

lib/cuckoo/core/database.py

@@ -119,6 +119,7 @@
 distconf = Config("distributed")
 web_conf = Config("web")
 LINUX_ENABLED = web_conf.linux.enabled
+LINUX_STATIC = web_conf.linux.static_only
 DYNAMIC_ARCH_DETERMINATION = web_conf.general.dynamic_arch_determination
 
 if repconf.mongodb.enabled:
@@ -1538,7 +1539,7 @@ def demux_sample_and_add_to_db(
                 package, _ = self._identify_aux_func(file_path, package, check_shellcode=check_shellcode)
 
         # extract files from the (potential) archive
-        extracted_files = demux_sample(file_path, package, options, platform=platform)
+        extracted_files, demux_error_msgs = demux_sample(file_path, package, options, platform=platform)
         # check if len is 1 and the same file, if diff register file, and set parent
         if extracted_files and (file_path, platform) not in extracted_files:
             sample_parent_id = self.register_sample(File(file_path), source_url=source_url)
@@ -1547,6 +1548,18 @@ def demux_sample_and_add_to_db(
 
         # create tasks for each file in the archive
         for file, platform in extracted_files:
+            # ToDo we lose package here and send APKs to windows
+            if platform in ("linux", "darwin") and LINUX_STATIC:
+                task_ids += self.add_static(
+                    file_path=file_path,
+                    priority=priority,
+                    tlp=tlp,
+                    user_id=user_id,
+                    username=username,
+                    options=options,
+                    package=package,
+                )
+                continue
             if static:
                 # On huge loads this just become a bottleneck
                 config = False
@@ -1621,6 +1634,8 @@ def demux_sample_and_add_to_db(
 
         if config and isinstance(config, dict):
             details = {"config": config.get("cape_config", {})}
+        if demux_error_msgs:
+            details["errors"] = demux_error_msgs
         # this is aim to return custom data, think of this as kwargs
         return task_ids, details
 
@@ -1694,7 +1709,7 @@ def add_static(
         user_id=0,
         username=False,
     ):
-        extracted_files = demux_sample(file_path, package, options)
+        extracted_files, demux_error_msgs = demux_sample(file_path, package, options)
         sample_parent_id = None
         # check if len is 1 and the same file, if diff register file, and set parent
         if not isinstance(file_path, bytes):

modules/processing/CAPE.py

@@ -113,7 +113,7 @@ def _cape_type_string(self, type_strings, file_info, append_file):
         elif type_strings[0] == "MS-DOS":
             file_info["cape_type"] = "DOS MZ image: executable"
         else:
-            file_info["cape_type"] = file_info["cape_type"] or "PE image"
+            file_info["cape_type"] = file_info["cape_type"] or "unknown"
         return append_file
 
     def _metadata_processing(self, metadata, file_info, append_file):

tests/test_demux.py

@@ -85,8 +85,9 @@ def test_demux_sample_pe32(self, grab_sample):
     def test_demux_package(self):
         empty_file = tempfile.NamedTemporaryFile()
 
-        assert demux.demux_sample(filename=empty_file.name, package="Emotet", options="foo", use_sflock=False) == [
-            (empty_file.name, "")
+        demuxed, _ = demux.demux_sample(filename=empty_file.name, package="Emotet", options="foo", use_sflock=False)
+        demuxed == [
+            (empty_file.name, "", "")
         ]
         empty_file.close()
 

tests/test_objects.py

@@ -223,7 +223,9 @@ def test_get_type(self, test_files):
         [
             ("temp_pe32", "PE32 executable (GUI) Intel 80386, for MS Windows", True),  # emulated magic type
             ("temp_pe64", "PE32+ executable (GUI) x86-64, for MS Windows", True),  # emulated magic type
-            ("temp_pe_aarch64", "MS-DOS executable PE32 executable Aarch64, for MS Windows", True),
+            # Broken we remove "MS-DOS executable"
+            # ("temp_pe_aarch64", "MS-DOS executable PE32 executable Aarch64, for MS Windows", True),
+            ("temp_pe_aarch64", "PE32 executable Aarch64, for MS Windows", True),
             ("temp_elf32", "ELF 32-bit LSB", False),
             ("temp_elf64", "ELF 64-bit LSB", False),
             ("temp_macho_arm64", "Mach-O 64-bit arm64 executable", False),

utils/submit.py

@@ -345,6 +345,7 @@ def main():
                 try:
                     tmp_path = store_temp_file(open(file_path, "rb").read(), sanitize_filename(os.path.basename(file_path)))
                     with db.session.begin():
+                        # ToDo expose extra_details["errors"]
                         task_ids, extra_details = db.demux_sample_and_add_to_db(
                             file_path=tmp_path,
                             package=args.package,

web/apiv2/views.py

@@ -187,7 +187,7 @@ def tasks_create_static(request):
     options = request.data.get("options", "")
     priority = force_int(request.data.get("priority"))
 
-    resp["error"] = False
+    resp["error"] = []
     files = request.FILES.getlist("file")
     extra_details = {}
     task_ids = []
@@ -203,6 +203,8 @@ def tasks_create_static(request):
                 user_id=request.user.id or 0,
             )
             task_ids.extend(task_id)
+            if extra_details.get("erros"):
+                resp["errors"].extend(extra_details["errors"])
         except CuckooDemuxError as e:
             resp = {"error": True, "error_value": e}
             return Response(resp)
@@ -226,7 +228,6 @@ def tasks_create_static(request):
                     resp["url"].append("{0}/submit/status/{1}".format(apiconf.api.get("url"), tid))
             else:
                 resp = {"error": True, "error_value": "Error adding task to database"}
-
     return Response(resp)
 
 
@@ -341,12 +342,22 @@ def tasks_create_file(request):
             if tmp_path:
                 details["path"] = tmp_path
                 details["content"] = content
-                status, task_ids_tmp = download_file(**details)
+                demux_error_msgs = []
+
+                result = download_file(**details)
+                if len(result) == 2:
+                    status, task_ids_tmp = result
+                elif len(result) == 3:
+                    status, task_ids_tmp, demux_error_msgs = result
+
                 if status == "error":
                     details["errors"].append({os.path.basename(tmp_path).decode(): task_ids_tmp})
                 else:
                     details["task_ids"] = task_ids_tmp
 
+                if demux_error_msgs:
+                    details["errors"].extend(demux_error_msgs)
+
         if details["task_ids"]:
             tasks_count = len(details["task_ids"])
         else:
@@ -565,12 +576,20 @@ def tasks_create_dlnexec(request):
             "user_id": request.user.id or 0,
         }
 
-        status, task_ids_tmp = download_file(**details)
+        result = download_file(**details)
+        if len(result) == 2:
+            status, task_ids_tmp = result
+        elif len(result) == 3:
+            status, task_ids_tmp, demux_error_msgs = result
+
         if status == "error":
             details["errors"].append({os.path.basename(path).decode(): task_ids_tmp})
         else:
             details["task_ids"] = task_ids_tmp
 
+        if demux_error_msgs:
+            details["errors"].extend(demux_error_msgs)
+
         if details["task_ids"]:
             tasks_count = len(details["task_ids"])
         else: