4.4.2 Release

Author: toddbruner

conversion/database_migration/bulk_alert_data.py

@@ -51,16 +51,18 @@ def transform_alert(alert=None, schema_id_map=None):
         if k =='columns' or k =='search' or k=='_raw':
             # We don't care about these columns because they should not show up in an alertgroup table
             continue
-        else: 
+        else:
             # Get the schem key id from the map we created beforehand
             schema_id = schema_id_map.get(f"{k}-{alertgroup_id}")
             if schema_id is None:
                 continue
             else:
                 data_value = alert['data'].get(k)
                 data_value_flaired = alert['data_with_flair'].get(k)
-                data_value = json.dumps(data_value)
-                data_value_flaired = json.dumps(data_value_flaired)
+                if not isinstance(data_value, str):
+                    data_value = json.dumps(data_value)
+                if not isinstance(data_value_flaired, str):
+                    data_value_flaired = json.dumps(data_value_flaired)
                 alert_data = [data_value, data_value_flaired, schema_id , alert['id']]
                 alert_datas.append(alert_data)
     return alert_datas

requirements.txt

@@ -1,24 +1,25 @@
-fastapi==0.112.1
+fastapi[standard]==0.115.12
 SQLAlchemy==2.0.32
 pydantic_core==2.23.4
 pydantic_settings==2.4.0
 pydantic==2.9.2
-mysql-connector-python==9.1.0
+mysqlclient==2.2.7
 meilisearch==0.31.4
 msal==1.30.0
 python-dateutil==2.9.0.post0
 nh3==0.2.18
-python-jose==3.3.0
+python-jose==3.4.0
 loguru==0.7.2
 email_validator==2.0.0
-Jinja2==3.1.4
+Jinja2==3.1.5
 ldap3==2.9.1
 passlib==1.7.4
-streaming-form-data==1.16.0
+streaming-form-data==1.19.0
 boto3==1.35.0
 tabulate==0.9.0
 markdownify==0.13.1
 xhtml2pdf==0.2.16
 pdf2docx==0.5.8
-sse-starlette==2.1.3
-python-multipart==0.0.9
+sse-starlette==2.2.1
+python-multipart==0.0.19
+pyzipper==0.3.6

setup.py

@@ -9,7 +9,7 @@ def get_requirements():
 
 setup(
     name="SCOT",
-    version="4.2.0",
+    version="4.4.0",
     description="Sandia Cyber Omni Tracker Server",
     packages=find_packages("src"),
     package_dir={"": "src"},

src/app/__init__.py

@@ -1 +1 @@
-__version__ = "4.0.0"
+__version__ = "4.4.0"

src/app/api/__init__.py

@@ -1,7 +1,16 @@
+from logging.config import dictConfig
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.openapi.utils import get_openapi
+from fastapi.openapi.docs import (
+    get_redoc_html,
+    get_swagger_ui_html
+)
+from fastapi.staticfiles import StaticFiles
 from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.responses import PlainTextResponse
+from sqlalchemy.exc import TimeoutError, DatabaseError
 
 from app.api.endpoints import router
 from app.core.config import settings
@@ -12,19 +21,27 @@ async def db_middleware(request: Request, call_next) -> None:
     """
     Middleware that wraps each API call in a transaction
     """
-    with SessionLocal() as db_session:
-        request.state.db = db_session
+    ROUTE_PREFIX_WHITELIST = ("/api/v1/firehose")
+    if request.url.path.startswith(ROUTE_PREFIX_WHITELIST):
+        return await call_next(request)
+    else:
+        try:
+            with SessionLocal() as db_session:
+                request.state.db = db_session
+                response = await call_next(request)
+                if response.status_code >= 500:
+                    db_session.rollback()
+                else:
 
-        response = await call_next(request)
-        if response.status_code >= 500:
-            db_session.rollback()
-        else:
+                    db_session.commit()
 
-            db_session.commit()
-
-            if hasattr(request.state, "audit_logger") and request.state.audit_logger:
-                request.state.audit_logger.save_audits(db_session)
-                db_session.commit()
+                    if hasattr(request.state, "audit_logger") and request.state.audit_logger:
+                        request.state.audit_logger.save_audits(db_session)
+                        db_session.commit()
+        except TimeoutError:
+            return TextResponse(503, "Timeout when connecting to database")
+        except DatabaseError:
+            return TextResponse(500, "Error when connecting to database")
 
     return response
 
@@ -33,18 +50,42 @@ def create_app() -> FastAPI:
     """
     :return:
     """
+    # Set up logging format
+    # dictConfig({
+    #    "version": 1,
+    #    "disable_existing_loggers": False,
+    #    "formatters": {
+    #        "access": {
+    #            "()": "uvicorn.logging.AccessFormatter",
+    #            "fmt": '%(asctime)s - %(levelprefix)s %(client_addr)s - \"%(request_line)s\" %(status_code)s',
+    #            "use_colors": True
+    #        },
+    #    },
+    #    "handlers": {
+    #        "access": {
+    #            "formatter": "access",
+    #            "class": "logging.StreamHandler",
+    #            "stream": "ext://sys.stdout",
+    #        },
+    #    },
+    #    "loggers": {
+    #        "uvicorn.access": {
+    #            "handlers": ["access"],
+    #            "level": "INFO",
+    #            "propagate": False
+    #        },
+    #    },
+    # })
+
     app = FastAPI(
         debug=settings.DEBUG,
-        title=settings.TITLE,
-        description=settings.DESCRIPTION,
-        docs_url=settings.DOCS_URL,
-        openapi_url=settings.OPENAPI_URL,
-        redoc_url=settings.REDOC_URL,
-        swagger_ui_parameters={
-            "filter": True
-        }
+        docs_url=None,
+        redoc_url=None,
+        openapi_url=settings.OPENAPI_URL
     )
 
+    app.mount('/api/static', StaticFiles(packages=[('app.api', 'static')]), name='static')
+
     app.include_router(router.api_router, prefix=settings.API_V1_STR)
 
     app.add_middleware(
@@ -56,13 +97,18 @@ def create_app() -> FastAPI:
         allow_headers=["*"],
     )
 
+    app.add_middleware(
+        GZipMiddleware,
+        minimum_size=500,
+        compresslevel=5)
+
     app.add_middleware(BaseHTTPMiddleware, dispatch=db_middleware)
 
     if not app.openapi_schema:
         # will need to update all links for open source stuff
         openapi_schema = get_openapi(
             title="SCOT",
-            version="4.2.0",
+            version="4.4.0",
             summary="Sandia Cyber Omni Tracker Server",
             description="The Sandia Cyber Omni Tracker (SCOT) is a cyber security incident response management system and knowledge base. Designed by cyber security incident responders, SCOT provides a new approach to manage security alerts, analyze data for deeper patterns, coordinate team efforts, and capture team knowledge. SCOT integrates with existing security applications to provide a consistent, easy to use interface that enhances analyst effectiveness.",
             routes=app.routes,
@@ -82,4 +128,24 @@ def create_app() -> FastAPI:
         }
         app.openapi_schema = openapi_schema
 
+    @app.get(settings.DOCS_URL, include_in_schema=False)
+    async def custom_swagger_ui_html():
+        return get_swagger_ui_html(
+            openapi_url=settings.OPENAPI_URL,
+            title=settings.TITLE,
+            swagger_js_url=settings.SWAGGER_JS_URL,
+            swagger_css_url=settings.SWAGGER_CSS_URL,
+            swagger_ui_parameters={
+                "filter": True
+            }
+        )
+
+    @app.get(settings.REDOC_URL, include_in_schema=False)
+    async def redoc_html():
+        return get_redoc_html(
+            openapi_url=settings.OPENAPI_URL,
+            title=settings.TITLE,
+            redoc_js_url=settings.REDOC_JS_URL,
+        )
+
     return app

src/app/api/deps.py

@@ -1,7 +1,8 @@
 import json
 from datetime import datetime, timedelta
+from typing import Annotated
 
-from fastapi import Depends, HTTPException, Request
+from fastapi import Depends, HTTPException, Request, Query, Path
 from fastapi.encoders import jsonable_encoder
 from jose import jwt
 from pydantic import ValidationError
@@ -77,6 +78,23 @@ def get_token_data(
     return token_data
 
 
+def get_token_data_no_apikey(
+    token: str = Depends(reusable_oauth2),
+) -> schemas.TokenPayload:
+    try:
+        if token:
+            payload = jwt.decode(
+                token, settings.SECRET_KEY, algorithms=[security.ALGORITHM]
+            )
+            token_data = schemas.TokenPayload(**payload)
+        else:
+            token_data = None
+    except (jwt.JWTError, ValidationError):
+        # Token validation failed; continue as if there's no token
+        token_data = None
+    return token_data
+
+
 def get_current_user(
     request: Request,
     db: Session = Depends(get_db),
@@ -247,7 +265,8 @@ class PermissionCheck:
     type_allow_whitelist = [TargetTypeEnum.entity, TargetTypeEnum.tag,
                             TargetTypeEnum.source, TargetTypeEnum.entity_class,
                             TargetTypeEnum.pivot, TargetTypeEnum.entity_type,
-                            TargetTypeEnum.none, None]
+                            TargetTypeEnum.special_metric, TargetTypeEnum.none,
+                            None]
 
     def __init__(self, target_type, permission, allow_admin=True):
         self.target_type = target_type
@@ -290,15 +309,26 @@ def check_permissions(self, db, roles, user, target_id=None):
 class PermissionCheckId(PermissionCheck):
     def __call__(
         self,
-        id: int,
+        id: Annotated[int, Path(...)],
         db: Session = Depends(get_db),
         user: models.User = Depends(get_current_active_user),
         roles: list[models.Role] = Depends(get_current_roles),
     ):
         if not self.check_permissions(db, roles, user, id):
-            raise HTTPException(
-                status_code=403,
-                detail="You do not have permission to access this resource, or "
-                "it does not exist",
-            )
+            raise HTTPException(403, "You do not have permission to access this resource, or it does not exist")
+        return True
+
+
+# Same as above, except for many objects
+class PermissionCheckIds(PermissionCheck):
+    def __call__(
+        self,
+        ids: Annotated[list[int], Query(...)],
+        db: Session = Depends(get_db),
+        user: models.User = Depends(get_current_active_user),
+        roles: list[models.Role] = Depends(get_current_roles),
+    ):
+        for id in ids:
+            if not self.check_permissions(db, roles, user, id):
+                raise HTTPException(403, "You do not have permission to access this resource, or it does not exist")
         return True

src/app/api/endpoints/alert.py

@@ -1,5 +1,5 @@
 from typing import Any, Annotated
-from fastapi import APIRouter, Depends, HTTPException, Body, Path
+from fastapi import APIRouter, Depends, HTTPException, Body, Path, Query
 from sqlalchemy.orm import Session
 from pydantic import ValidationError
 
@@ -37,6 +37,27 @@
 description, examples = create_schema_details(schemas.AlertUpdate)
 
 
+@router.put(
+    "/many",
+    response_model=list[schemas.Alert],
+    summary="Update many alerts",
+    description=description
+)
+def update_alerts(
+    *,
+    db: Session = Depends(deps.get_db),
+    audit_logger: deps.AuditLogger = Depends(deps.get_audit_logger),
+    current_user: models.User = Depends(deps.get_current_active_user),
+    current_roles: list[models.Role] = Depends(deps.get_current_roles),
+    ids: Annotated[list[int], Query(...)],
+    obj: Annotated[schemas.AlertUpdate, Body(..., openapi_examples=examples)],
+) -> Any:
+    _objs = []
+    for id in ids:
+        _objs.append(update_alert(db=db, audit_logger=audit_logger, current_user=current_user, current_roles=current_roles, id=id, obj=obj))
+    return _objs
+
+
 # Custom PUT so that you can modify alerts if you have access to the alertgroup
 @router.put(
     "/{id}",

src/app/api/endpoints/alertgroup.py

@@ -19,24 +19,26 @@
     generic_search,
     generic_export,
     generic_upvote_and_downvote,
-    generic_user_links
+    generic_user_links,
+    generic_tag_untag
 )
 
 router = APIRouter()
 
 # Create get, post, put, and delete endpoints
-generic_export(router, crud.alert_group, TargetTypeEnum.alertgroup)
 generic_get(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
 generic_post(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed, schemas.AlertGroupDetailedCreate)
 generic_put(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed, schemas.AlertGroupUpdate)
 generic_delete(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
+generic_search(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupSearch, schemas.AlertGroup)
 generic_undelete(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
 generic_entities(router, TargetTypeEnum.alertgroup)
 generic_history(router, crud.alert_group, TargetTypeEnum.alertgroup)
 generic_reflair(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
-generic_search(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupSearch, schemas.AlertGroup)
+generic_export(router, crud.alert_group, TargetTypeEnum.alertgroup)
 generic_upvote_and_downvote(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroup)
 generic_user_links(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroupDetailed)
+generic_tag_untag(router, crud.alert_group, TargetTypeEnum.alertgroup, schemas.AlertGroup)
 
 
 alertgroup_read_dep = Depends(deps.PermissionCheckId(TargetTypeEnum.alertgroup, PermissionEnum.read))