Fix importer bugs

releases/unreleased/importer-failed-to-reschedule-tasks.yml

@@ -0,0 +1,9 @@
+---
+title: Importer failed to reschedule tasks
+category: fixed
+author: Jose Javier Merchante <[email protected]>
+issue: null
+notes: >
+  Fix a bug that prevented the importer from rescheduling tasks
+  because `from_date` was not JSON serializable when inserting into
+  the database.

releases/unreleased/importing-identities-with-oversized-fields-failed.yml

@@ -0,0 +1,8 @@
+---
+title: Importing identities with oversized fields failed
+category: fixed
+author: Jose Javier Merchante <[email protected]>
+issue: null
+notes: >
+  Fix a bug where importing identities with a large field (e.g., username)
+  caused the whole process to fail. Now, such identities are skipped instead.

sortinghat/core/importer/backend.py

@@ -21,6 +21,8 @@
 
 import logging
 
+from django.db import DataError
+
 import sortinghat.core.importer.backends
 from grimoirelab_toolkit.introspect import inspect_signature_parameters
 from .. import api
@@ -151,6 +153,8 @@ def __load_identities(self, identities):
                 nidentities += 1
             except InvalidValueError as e:
                 logger.warning(str(e))
+            except DataError as e:
+                logger.warning(f"Data error inserting identity: {e}")
             except AlreadyExistsError as e:
                 stored_identity = Identity.objects.get(source=identity.source,
                                                        email=identity.email,

sortinghat/core/jobs.py

@@ -952,8 +952,8 @@ def on_success_job(job, connection, result, *args, **kwargs):
         backends = find_import_identities_backends()
         backend_name = task.args['backend_name']
         if 'from_date' in backends[backend_name]['args']:
-            job.kwargs['from_date'] = task.scheduled_datetime
-            task.args['from_date'] = task.scheduled_datetime
+            job.kwargs['from_date'] = task.scheduled_datetime.isoformat()
+            task.args['from_date'] = task.scheduled_datetime.isoformat()
 
     if not task.interval:
         logger.info("Interval not defined, not rescheduling task.")