Skip to content

Add clear and update mode for csv:import #2256

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
danloveg wants to merge 5 commits into
base: qa/2.x
Choose a base branch
from
Draft

Add clear and update mode for csv:import #2256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5761e82
Add clear-and-update mode (noop for now)
danloveg Nov 18, 2025
bd565dd
Add guard to only clear QubitInformationObjects
danloveg Nov 18, 2025
d1e707a
Implement clear-and-update mode
danloveg Nov 19, 2025
a1b0f3c
Delete objects unless --keep-digital-objects
danloveg Dec 5, 2025
806198e
Update task description, add warning
danloveg Dec 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 148 additions & 4 deletions lib/QubitFlatfileImport.class.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ class QubitFlatfileImport
public $searchIndexingDisabled = true; // disable per-object search indexing by default
public $disableNestedSetUpdating = false; // update nested set on object creation
public $matchAndUpdate = false; // match existing records & update them
public $clearAndUpdate = false; // clear matching records & update them
public $deleteAndReplace = false; // delete matching records & replace them
public $skipMatched = false; // skip creating new record if matching one is found
public $skipUnmatched = false; // skip creating new record if matching one is not found
Expand Down Expand Up @@ -67,6 +68,10 @@ class QubitFlatfileImport

// Replaceable logic to filter content before entering Qubit
public $contentFilterLogic;
public $contentLogic;

// The object being imported/updated
public $object;

public function __construct($options = [])
{
Expand Down Expand Up @@ -135,6 +140,13 @@ public function setUpdateOptions($options)

break;

case 'clear-and-update':
// Clear matching records before updating them in-place
$this->clearAndUpdate = true;
$this->keepDigitalObjects = $options['keep-digital-objects'];

break;

case 'match-and-update':
// Save match option. If update is ON, and match is set, only updating
// existing records - do not create new objects.
Expand Down Expand Up @@ -537,7 +549,7 @@ public function row($row = [])

public function isUpdating()
{
return $this->matchAndUpdate || $this->deleteAndReplace;
return $this->matchAndUpdate || $this->clearAndUpdate || $this->deleteAndReplace;
}

/**
Expand Down Expand Up @@ -929,7 +941,7 @@ public function createOrFetchAndUpdateActorForIo($name, $options = [])
}

// Change actor history when updating a match in the same repo
if ($this->matchAndUpdate) {
if ($this->matchAndUpdate || $this->clearAndUpdate) {
$actor->history = $options['history'];
$actor->save();

Expand Down Expand Up @@ -1914,6 +1926,10 @@ private function handleInformationObjectRow()
$this->handleDeleteAndReplace();
}

if ($this->clearAndUpdate) {
$this->handleClearAndUpdate();
}

// Execute ad-hoc row pre-update logic (remove related data, etc.)
$this->executeClosurePropertyIfSet('updatePreparationLogic');
$skipRowProcessing = false;
Expand Down Expand Up @@ -1941,6 +1957,9 @@ private function getActionDescription()
if ($this->matchAndUpdate) {
return 'updating in place';
}
if ($this->clearAndUpdate) {
return 'clearing and updating in place';
}

return 'skipping';
}
Expand All @@ -1962,6 +1981,131 @@ private function handleDeleteAndReplace()
$this->object->slug = $oldSlug; // Retain previous record's slug
}

/**
* Clear the content of the given object to prepare it to be re-defined in place. This enables
* all fields of information to be re-written without having to delete the object.
*
* Clears:
* - Direct properties of the object
* - Properties on i18n objects for the selected culture
*
* Deletes:
* - Related QubitObjectTermRelation objects
* - Related QubitProperty objects
* - QubitRelation objects where this is the "Object" part of the relationship
* - QubitRelation objects where this is the "Subject" part of the relationship (except for
* related description relationships which can't be imported via CSV)
*/
private function handleClearAndUpdate()
{
$directProperties = [];

if ($this->object instanceof QubitInformationObject) {
$directProperties = [
'descriptionIdentifier',
'descriptionDetailId',
'descriptionStatusId',
'levelOfDescriptionId',
'repositoryId',
];
} else {
throw new sfException(
'Cannot handle clear-and-update for objects that are not QubitInformationObject! Got: '.get_class($this->object)
);
}

// Clear all properties that exist on the object itself
// e.g., Description identifier, level of description
foreach ($directProperties as $directProperty) {
$this->object->{$directProperty} = null;
}

// Clear i18n object for the given culture
// e.g., Title, Scope and content
$culture = $this->columnValue('culture');
$i18ns = $this->object->informationObjectI18ns->indexBy('culture');

if (isset($i18ns[$culture])) {
$i18n = $i18ns[$culture];

foreach ($this->standardColumns as $column) {
if (in_array($column, ['createdAt', 'updatedAt', 'culture'])) {
continue;
}
$i18n->{$column} = null;
}
}

// Remove all object-term relations
// e.g., Place access points, name access points
$criteria = new Criteria();
$criteria->add(QubitObjectTermRelation::OBJECT_ID, $this->object->id);
$objectTermRelations = QubitObjectTermRelation::get($criteria);

foreach ($objectTermRelations as $objectTermRelation) {
$objectTermRelation->delete();
}

// Remove all notes
// e.g., Archivist note, Credits note
$criteria = new Criteria();
$criteria->add(QubitNote::OBJECT_ID, $this->object->id);
$notes = QubitNote::get($criteria);

foreach ($notes as $note) {
$note->delete();
}

// Remove all events
$criteria = new Criteria();
$criteria->add(QubitEvent::OBJECT_ID, $this->object->id);
$events = QubitEvent::get($criteria);

foreach ($events as $event) {
$event->delete();
}

// Remove all special properties stored as QubitProperty objects
// e.g., Script of description, Alternative identifiers
$properties = $this->object->getProperties();

foreach ($properties as $property) {
$property->delete();
}

// Remove relationships where the relationship terminates at this object
// e.g., Physical object -> has -> Information object
$criteria = new Criteria();
$criteria = $this->object->addrelationsRelatedByobjectIdCriteria($criteria);
$objectRelations = QubitRelation::get($criteria);

foreach ($objectRelations as $relation) {
$relation->delete();
}

// Remove relationships where the relationship originates from this object
// e.g., Information object -> has -> Accession object
$criteria = new Criteria();
$criteria = $this->object->addrelationsRelatedBysubjectIdCriteria($criteria);
$subjectRelations = QubitRelation::get($criteria);

foreach ($subjectRelations as $relation) {
// There is no way to import this type of relationship, so skip removing it
if (QubitTerm::RELATED_MATERIAL_DESCRIPTIONS_ID == $relation->typeId) {
continue;
}

$relation->delete();
}

// Remove digital object unless --keep-digital-objects is set
if (!$this->keepDigitalObjects) {
if (null !== $do = $this->object->getDigitalObject()) {
$do->delete();
}
}
}

/**
* Creates a new information object if --skip-unmatched isn't set in options.
*/
Expand Down Expand Up @@ -2130,8 +2274,8 @@ private function handleRepositoryAndActorRow()
// Execute ad-hoc row pre-update logic (remove related data, etc.)
$this->executeClosurePropertyIfSet('updatePreparationLogic');

// Match and update: update current object
if ($this->matchAndUpdate) {
// Match and update & clear and update: update current object
if ($this->matchAndUpdate || $this->clearAndUpdate) {
return false;
}

Expand Down
22 changes: 20 additions & 2 deletions lib/task/import/csvImportBaseTask.class.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
*/
abstract class csvImportBaseTask extends arBaseTask
{
// Some classes may not implement this method, enable this feature only if the sub-class explicitly allows for it
protected bool $enableClearAndUpdate = false;

/**
* If updating, delete existing digital object if updating, a path or UI has
* been specified, and not keeping digial objects.
Expand Down Expand Up @@ -490,11 +493,22 @@ protected function validateOptions($options)
throw new sfException('The --limit option requires the --update option to be present.');
}

if ($options['keep-digital-objects'] && 'match-and-update' != trim($options['update'])) {
throw new sfException('The --keep-digital-objects option can only be used when --update=\'match-and-update\' option is present.');
if ($options['keep-digital-objects']) {
$updateMode = trim($options['update']);

if (!$this->enableClearAndUpdate && 'match-and-update' != $updateMode) {
throw new sfException('The --keep-digital-objects option can only be used when --update=\'match-and-update\' option is present.');
}
if ($this->enableClearAndUpdate && !array_search($updateMode, ['match-and-update', 'clear-and-update'])) {
throw new sfException('The --keep-digital-objects option can only be used when the --update=\'match-and-update\' or --update=\'clear-and-update\' option is present.');
}
}

$this->validateUpdateOptions($options);

if ($this->enableClearAndUpdate && 'clear-and-update' === trim($options['update'])) {
echo "WARNING: The clear-and-update import mode is experimental.\n";
}
}

/**
Expand All @@ -510,6 +524,10 @@ protected function validateUpdateOptions($options)

$validParams = ['match-and-update', 'delete-and-replace'];

if ($this->enableClearAndUpdate) {
$validParams[] = 'clear-and-update';
}

if (!in_array(trim($options['update']), $validParams)) {
$msg = sprintf('Parameter "%s" is not valid for --update option. ', $options['update']);
$msg .= sprintf('Valid options are: %s', implode(', ', $validParams));
Expand Down
9 changes: 6 additions & 3 deletions lib/task/import/csvImportTask.class.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,15 @@
*/
class csvImportTask extends csvImportBaseTask
{
// Enable clearing and updating
protected bool $enableClearAndUpdate = true;

protected $namespace = 'csv';
protected $name = 'import';
protected $briefDescription = 'Import csv information object data';

protected $detailedDescription = <<<'EOF'
Import CSV data
Import new or update existing information objects via CSV
EOF;

/**
Expand Down Expand Up @@ -913,7 +916,7 @@ protected function configure()
'update',
null,
sfCommandOption::PARAMETER_REQUIRED,
'Attempt to update if description has already been imported. Valid option values are "match-and-update" & "delete-and-replace".'
'Attempt to update if description has already been imported. Valid option values are "match-and-update", "clear-and-update", & "delete-and-replace".'
),
new sfCommandOption(
'skip-matched',
Expand Down Expand Up @@ -950,7 +953,7 @@ protected function configure()
'keep-digital-objects',
null,
sfCommandOption::PARAMETER_NONE,
'Skip the deletion of existing digital objects and their derivatives when using --update with "match-and-update".'
'Skip the deletion of existing digital objects and their derivatives when using --update with "match-and-update" or "clear-and-update".'
),
new sfCommandOption(
'roundtrip',
Expand Down
Loading
Loading