4.3 Release

Author: toddbruner

conversion/conversion.md

@@ -2,6 +2,11 @@
 
 This directory contains scripts for migrating data from version 3 of SCOT to version 4. They are grouped into three categories: database migrations, file migrations, and extra (optional) migrations. Three bash shell scripts have been provided for you to run the applicable migrations in each category.
 
+## Necessary tools to run conversion
+You will need the following installed to run conversions:
+- python 3.11 with the contents of requirements-test.txt from this repo installed via pip
+- mysql-shell
+
 ## Database Conversion
 This set of scripts migrates the core database data to the SCOT4 database by pulling data directly from the SCOT3 mongodb database. Almost all SCOT3 installations migrating to SCOT4 will want to do this. The `database_conversion.sh` script will run all of the necessary scripts for you.
 
@@ -45,4 +50,4 @@ The following environment variables should be set when running `file_conversion.
  - SCOT_ADMIN_APIKEY - a SCOT4 API key with admin priveleges (see above for one way to create one)
  - SCOT3_FILE_PREFIX (needed for file migration) - the directory under which the files were stored in the SCOT3 database, this defaults to the default in SCOT3, which was `/opt/scotfiles/`
  - SCOT_FILES_DIR (needed for file migration) - the directory on the current machine in which the old SCOT3 files are stored (with the same file structure that the SCOT3 installation had)
- - SCOT_CACHED_IMAGES_DIR (needed for cached images migration) - the directory on the current machine that contains the SCOT3 cached images in their original file structure (this is usually the /cached_images/ directory in the SCOT3 files)
+ - SCOT_CACHED_IMAGES_DIR (needed for cached images migration) - the directory on the current machine that contains the SCOT3 cached images in their original file structure (this is usually the /cached_images/ directory in the SCOT3 files)

conversion/database_migration/bulk_entries.py

@@ -73,6 +73,9 @@ def main(mongo_db=None, role_lookup=None):
         with tqdm.tqdm(total=scot3_entry_count) as pbar:
             bulk_array = []
             for entry in scot3_entries:
+                if entry.get('target') is None or entry['target'].get('type') is None:
+                    pbar.update(1) # Malformed data
+                    continue
                 if entry['target']['type'] == 'alert' or entry['target']['type'] == 'alertgroup' or entry['target']['type'] == 'feed':
                     pbar.update(1)
                     continue

conversion/database_migration/bulk_links.py

@@ -14,6 +14,9 @@ def main(mongo_db=None):
     scot3_links = mongo_db.link.find()
     with tqdm.tqdm(total=scot3_link_count) as pbar:
         for link in scot3_links:
+            if link.get('vertices') is None:
+                pbar.update(1)
+                continue
             if type(link.get('when')) is int:
                 new_link = [ datetime.fromtimestamp(link['when']).astimezone(timezone.utc).replace(tzinfo=None), datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None), link['id'], link['vertices'][1]['type'], link['vertices'][1]['id'], link['vertices'][0]['type'], link['vertices'][0]['id']]
             else:

conversion/database_migration/bulk_roles.py

@@ -28,9 +28,9 @@ def main(mongo_db=None):
         for count, role in enumerate(tqdm.tqdm(unique_roles)):
             if role == '':
                 continue
-            role_row = [count+2, role, 'migrated from SCOT3', datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None), datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None)] #Using an offset of 3 since we have two default groups: admin and everyone, created by default and taking up role ids 1 & 2 respectively
+            role_row = [count+3, role, 'migrated from SCOT3', datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None), datetime.fromtimestamp(0).astimezone(timezone.utc).replace(tzinfo=None)] #Using an offset of 3 since we have two default groups: admin and everyone, created by default and taking up role ids 1 & 2 respectively
             writer.writerow(role_row)
-            role_lookup[role] = count+2
+            role_lookup[role] = count+3
 
     return role_lookup
 

conversion/database_migration/conversion_utilities.py

@@ -22,6 +22,6 @@ def write_tag_source_links(thing=None, thing_type=None, tag_lookup=None, source_
         if thing_type == "signature":
             # Check if reference ID exists here
             target_data = thing.get('data').get('target')
-            if target_data is not None:
+            if target_data is not None and target_data.get('type') is not None and target_data.get('id') is not None:
                 new_link = [thing_type, thing.get('id'), target_data['type'], target_data['id']]
                 link_csv_writer.writerow(new_link)

conversion/database_migration/initial_scot4_database.sql

@@ -271,6 +271,10 @@ CREATE TABLE `auth_settings` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
+LOCK TABLES `auth_settings` WRITE;
+INSERT INTO `auth_settings` VALUES (1, 'local', '{}', 1, '2023-06-06 16:53:27','2023-06-06 16:53:27');
+UNLOCK TABLES;
+
 --
 -- Table structure for table `auth_storage`
 --
@@ -840,9 +844,9 @@ LOCK TABLES `links` WRITE;
 /*!40000 ALTER TABLE `links` ENABLE KEYS */;
 UNLOCK TABLES;
 
----
---- Table structure for table `metrics`
----
+--
+-- Table structure for table `metrics`
+--
 
 DROP TABLE IF EXISTS `metrics`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;

conversion/database_migration/scot3_scot4_tsv_import.py

@@ -74,4 +74,4 @@
     util.import_table(f"{staging_directory}/entity_class_associations.csv", {"table": "entity_class_entity_association", "columns": ['entity_id', 'entity_class_id'], "dialect": "tsv", 'fieldsEscapedBy': '\t', 'fieldsEnclosedBy': "'", 'fieldsEscapedBy': '\0', 'linesTerminatedBy': "\n", "skipRows": 1, "showProgress": True})    
 
     if os.path.isfile("./games.csv"):
-        util.import_table("./games.csv", { "table": "games", "columns": [], "dialect": "tsv", 'fieldsEscapedBy': '\t', 'fieldsEnclosedBy': "'", 'fieldsEscapedBy': '\0', 'linesTerminatedBy': "\n", "skipRows": 1, "showProgress": True})
+        util.import_table("./games.csv", { "table": "games", "columns": ["game_id", "game_name", "tooltip", "results", "created_date", "modified_date"], "dialect": "tsv", 'fieldsEscapedBy': '\t', 'fieldsEnclosedBy': "'", 'fieldsEscapedBy': '\0', 'linesTerminatedBy': "\n", "skipRows": 1, "showProgress": True})

conversion/extra_migration/update_admin_password_and_api_key.py

@@ -8,7 +8,7 @@ def main(db_session):
     new_admin_apikey = os.getenv('SCOT_ADMIN_APIKEY')
     db_obj = db_session.query(User).filter(User.username=='scot-admin').one_or_none()
     if new_admin_pw:
-        update_dict={'username':'scot-admin', 'password':os.environ['SCOT_ADMIN_PASSWORD'])
+        update_dict={'username':'scot-admin', 'password':os.environ['SCOT_ADMIN_PASSWORD']}
         crud.user.update(db_session=db_session, db_obj=db_obj, obj_in=update_dict)
     else:
         print('SCOT_ADMIN_PASSWORD not set, not resetting scot-admin password')

conversion/file_migration/migrate_cached_images.py

@@ -9,6 +9,9 @@
 import pathlib
 import re
 import urllib3
+
+from app.db.session import SessionLocal
+from app.models import Entry
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 def rewrite_cached_images(html, flaired_html, entry_id):

requirements-test.txt

@@ -7,4 +7,5 @@ tqdm==4.66.5
 pymongo==4.8.0
 splunk-sdk==2.0.2
 pytest-xdist==3.6.1
+Faker==27.0.0
 -r requirements.txt