demux

Author: doomedraven

lib/cuckoo/common/demux.py

@@ -32,7 +32,8 @@
 cuckoo_conf = Config()
 web_cfg = Config("web")
 tmp_path = cuckoo_conf.cuckoo.get("tmppath", "/tmp")
-linux_enabled = web_cfg.linux.get("enabled", False)
+# ToDo allow_static
+linux_enabled = web_cfg.linux.get("enabled", False) or web_cfg.linux.get("static_only", False)
 
 demux_extensions_list = {
     b".accdr",
@@ -162,7 +163,8 @@ def is_valid_package(package: str) -> bool:
     return any(ptype in package for ptype in VALID_PACKAGES)
 
 
-def _sf_children(child: sfFile) -> bytes:
+# ToDo fix return type
+def _sf_children(child: sfFile):  # -> bytes:
     path_to_extract = ""
     _, ext = os.path.splitext(child.filename)
     ext = ext.lower()
@@ -184,15 +186,17 @@ def _sf_children(child: sfFile) -> bytes:
                 _ = path_write_file(path_to_extract, child.contents)
         except Exception as e:
             log.error(e, exc_info=True)
-    return path_to_extract.encode()
+    return (path_to_extract.encode(), child.platform, child.get_type(), child.get_size())
 
 
-def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) -> List[bytes]:
+# ToDo fix typing need to add str as error msg
+def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True):  # -> List[bytes]:
     retlist = []
     # do not extract from .bin (downloaded from us)
     if os.path.splitext(filename)[1] == b".bin":
-        return retlist
+        return retlist, ""
 
+    # ToDo need to introduce error msgs here
     try:
         password = options2passwd(options) or "infected"
         try:
@@ -201,9 +205,13 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) ->
             unpacked = unpack(filename, check_shellcode=check_shellcode)
 
         if unpacked.package in whitelist_extensions:
-            return [filename]
+            file = File(filename)
+            magic_type = file.get_type()
+            platform = file.get_platform()
+            file_size = file.get_size()
+            return [filename, platform, magic_type, file_size], ""
         if unpacked.package in blacklist_extensions:
-            return [filename]
+            return [], "blacklisted package"
         for sf_child in unpacked.children:
             if sf_child.to_dict().get("children"):
                 retlist.extend(_sf_children(ch) for ch in sf_child.children)
@@ -214,7 +222,7 @@ def demux_sflock(filename: bytes, options: str, check_shellcode: bool = True) ->
                 retlist.append(_sf_children(sf_child))
     except Exception as e:
         log.error(e, exc_info=True)
-    return list(filter(None, retlist))
+    return list(filter(None, retlist)), ""
 
 
 def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool = True, platform: str = ""):  # -> tuple[bytes, str]:
@@ -227,21 +235,29 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
     if isinstance(filename, str) and use_sflock:
         filename = filename.encode()
 
+    error_list = []
     retlist = []
     # if a package was specified, trim if allowed and required
     if package:
-
         if package in ("msix",):
             retlist.append((filename, "windows"))
         else:
             if File(filename).get_size() <= web_cfg.general.max_sample_size or (
                 web_cfg.general.allow_ignore_size and "ignore_size_check" in options
             ):
-                retlist.append((filename, platform))
+                retlist.append((filename, platform, ""))
             else:
                 if web_cfg.general.enable_trim and trim_file(filename):
                     retlist.append((trimmed_path(filename), platform))
-        return retlist
+                else:
+                    error_list.append(
+                        {
+                            os.path.basename(
+                                filename
+                            ): "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option"
+                        }
+                    )
+        return retlist, error_list
 
     # handle quarantine files
     tmp_path = unquarantine(filename)
@@ -259,9 +275,16 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
         if use_sflock:
             if HAS_SFLOCK:
                 retlist = demux_office(filename, password, platform)
-                return retlist
+                return retlist, error_list
             else:
                 log.error("Detected password protected office file, but no sflock is installed: poetry install")
+                error_list.append(
+                    {
+                        os.path.basename(
+                            filename
+                        ): "Detected password protected office file, but no sflock is installed or correct password provided"
+                    }
+                )
 
     # don't try to extract from Java archives or executables
     if (
@@ -279,6 +302,13 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
         else:
             if web_cfg.general.enable_trim and trim_file(filename):
                 retlist.append((trimmed_path(filename), platform))
+            else:
+                error_list.append(
+                    {
+                        os.path.basename(filename),
+                        "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option",
+                    }
+                )
         return retlist
 
     new_retlist = []
@@ -288,26 +318,34 @@ def demux_sample(filename: bytes, package: str, options: str, use_sflock: bool =
         check_shellcode = False
 
     # all in one unarchiver
-    retlist = demux_sflock(filename, options, check_shellcode) if HAS_SFLOCK and use_sflock else []
+    retlist, error_msg = demux_sflock(filename, options, check_shellcode) if HAS_SFLOCK and use_sflock else []
     # if it isn't a ZIP or an email, or we aren't able to obtain anything interesting from either, then just submit the
     # original file
     if not retlist:
+        if error_msg:
+            error_list.append({os.path.basename(filename), error_msg})
         new_retlist.append((filename, platform))
     else:
-        for filename in retlist:
+        for filename, platform, magic_type, file_size in retlist:
             # verify not Windows binaries here:
-            file = File(filename)
-            magic_type = file.get_type()
-            platform = file.get_platform()
             if platform == "linux" and not linux_enabled and "Python" not in magic_type:
+                error_list.append({os.path.basename(filename): "Linux processing is disabled"})
                 continue
 
-            if file.get_size() > web_cfg.general.max_sample_size and not (
+            if file_size > web_cfg.general.max_sample_size and not (
                 web_cfg.general.allow_ignore_size and "ignore_size_check" in options
             ):
                 if web_cfg.general.enable_trim:
                     # maybe identify here
                     if trim_file(filename):
                         filename = trimmed_path(filename)
+            else:
+                error_list.append(
+                    {
+                        os.path.basename(filename),
+                        "File too bit, enable 'allow_ignore_size' in web.conf or use 'ignore_size_check' option",
+                    }
+                )
             new_retlist.append((filename, platform))
-    return new_retlist[:10]
+
+    return new_retlist[:10], error_list

lib/cuckoo/common/web_utils.py

@@ -591,7 +591,6 @@ def download_file(**kwargs):
     onesuccess = False
 
     username = False
-    demux_error_msgs = []
     """
     put here your custom username assignation from your custom auth, Ex:
     request_url = kwargs["request"].build_absolute_uri()
@@ -719,14 +718,12 @@ def download_file(**kwargs):
     if not static and "dist_extract" in kwargs["options"]:
         static = True
 
-    demux_error_msgs = []
-
     for machine in kwargs.get("task_machines", []):
         if machine == "first":
             machine = None
 
         # Keep this as demux_sample_and_add_to_db in DB
-        task_ids_new, extra_details, demux_error_msgs = db.demux_sample_and_add_to_db(
+        task_ids_new, extra_details = db.demux_sample_and_add_to_db(
             file_path=kwargs["path"],
             package=package,
             timeout=timeout,
@@ -769,7 +766,7 @@ def download_file(**kwargs):
     if not onesuccess:
         return "error", {"error": f"Provided hash not found on {kwargs['service']}"}
 
-    return "ok", kwargs["task_ids"], demux_error_msgs
+    return "ok", kwargs["task_ids"], extra_details.get("erros", [])
 
 
 def save_script_to_storage(task_ids, kwargs):

lib/cuckoo/core/database.py

@@ -119,6 +119,7 @@
 distconf = Config("distributed")
 web_conf = Config("web")
 LINUX_ENABLED = web_conf.linux.enabled
+LINUX_STATIC = web_conf.linux.static_only
 DYNAMIC_ARCH_DETERMINATION = web_conf.general.dynamic_arch_determination
 
 if repconf.mongodb.enabled:
@@ -1538,7 +1539,7 @@ def demux_sample_and_add_to_db(
                 package, _ = self._identify_aux_func(file_path, package, check_shellcode=check_shellcode)
 
         # extract files from the (potential) archive
-        extracted_files = demux_sample(file_path, package, options, platform=platform)
+        extracted_files, demux_error_msgs = demux_sample(file_path, package, options, platform=platform)
         # check if len is 1 and the same file, if diff register file, and set parent
         if extracted_files and (file_path, platform) not in extracted_files:
             sample_parent_id = self.register_sample(File(file_path), source_url=source_url)
@@ -1547,6 +1548,18 @@ def demux_sample_and_add_to_db(
 
         # create tasks for each file in the archive
         for file, platform in extracted_files:
+            # ToDo we lose package here and send APKs to windows
+            if platform in ("linux", "darwin") and LINUX_STATIC:
+                task_ids += self.add_static(
+                    file_path=file_path,
+                    priority=priority,
+                    tlp=tlp,
+                    user_id=user_id,
+                    username=username,
+                    options=options,
+                    package=package,
+                )
+                continue
             if static:
                 # On huge loads this just become a bottleneck
                 config = False
@@ -1621,6 +1634,8 @@ def demux_sample_and_add_to_db(
 
         if config and isinstance(config, dict):
             details = {"config": config.get("cape_config", {})}
+        if demux_error_msgs:
+            details["errors"] = demux_error_msgs
         # this is aim to return custom data, think of this as kwargs
         return task_ids, details
 

utils/submit.py

@@ -345,6 +345,7 @@ def main():
                 try:
                     tmp_path = store_temp_file(open(file_path, "rb").read(), sanitize_filename(os.path.basename(file_path)))
                     with db.session.begin():
+                        # ToDo expose extra_details["errors"]
                         task_ids, extra_details = db.demux_sample_and_add_to_db(
                             file_path=tmp_path,
                             package=args.package,

web/apiv2/views.py

@@ -187,15 +187,14 @@ def tasks_create_static(request):
     options = request.data.get("options", "")
     priority = force_int(request.data.get("priority"))
 
-    resp["error"] = False
+    resp["error"] = []
     files = request.FILES.getlist("file")
     extra_details = {}
     task_ids = []
-    demux_error_msgs = []
     for sample in files:
         tmp_path = store_temp_file(sample.read(), sanitize_filename(sample.name))
         try:
-            task_id, extra_details, demux_error_msg = db.demux_sample_and_add_to_db(
+            task_id, extra_details = db.demux_sample_and_add_to_db(
                 tmp_path,
                 options=options,
                 priority=priority,
@@ -204,8 +203,8 @@ def tasks_create_static(request):
                 user_id=request.user.id or 0,
             )
             task_ids.extend(task_id)
-            if demux_error_msg:
-                demux_error_msgs.extend(demux_error_msg)
+            if extra_details.get("erros"):
+                resp["errors"].extend(extra_details["errors"])
         except CuckooDemuxError as e:
             resp = {"error": True, "error_value": e}
             return Response(resp)
@@ -229,8 +228,6 @@ def tasks_create_static(request):
                     resp["url"].append("{0}/submit/status/{1}".format(apiconf.api.get("url"), tid))
             else:
                 resp = {"error": True, "error_value": "Error adding task to database"}
-    if demux_error_msgs:
-        resp["errors"].extend(demux_error_msgs)
     return Response(resp)