From 189d0075720c4aafaa622de3611310ed1c596cd0 Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Wed, 23 Sep 2026 10:41:43 +0100 Subject: [PATCH] feat(totem): add TOTEM model and stream --- .../rdm/records/transform/models/_config.py | 7 +++- .../rdm/records/transform/models/totem.py | 32 +++++++++++++++++++ .../xml_processing/rules/research.py | 27 ++++++++++++++++ .../transform/xml_processing/rules/ship.py | 26 --------------- cds_migrator_kit/rdm/streams.yaml | 11 +++++++ setup.cfg | 5 +++ 6 files changed, 81 insertions(+), 27 deletions(-) create mode 100644 cds_migrator_kit/rdm/records/transform/models/totem.py diff --git a/cds_migrator_kit/rdm/records/transform/models/_config.py b/cds_migrator_kit/rdm/records/transform/models/_config.py index 501047e3..3f229cd5 100644 --- a/cds_migrator_kit/rdm/records/transform/models/_config.py +++ b/cds_migrator_kit/rdm/records/transform/models/_config.py @@ -34,9 +34,13 @@ "540__3", # Material of the license "540__9", # Also material of the license "540__g", # From SIS (see decision log) + "541__a", # See decision log + "541__c", # See decision log + "541__d", # See decision log + "541__e", # See decision log "542__3", # Also material of the license "595__i", # From SIS: see decision log - "696__9", # From SIS: see decision log + "695__9", # From SIS: see decision log "700__m", # email of contributor "773__t", # from SIS: can be ignored "773__0", # from SIS: can be ignored @@ -58,6 +62,7 @@ "961__h", # CDS modification tag # TODO "961__l", # CDS modification tag # TODO "961__x", # CDS modification tag # TODO + "964__a", # See decision log "981__a", # duplicate record id "999C50", # From SIS: all 999* fields can be ignored "999C52", # https://cds.cern.ch/record/2640188/export/hm?ln=en diff --git a/cds_migrator_kit/rdm/records/transform/models/totem.py b/cds_migrator_kit/rdm/records/transform/models/totem.py new file mode 100644 index 00000000..6c4dbcff --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/totem.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM TOTEM model.""" + +from cds_migrator_kit.rdm.records.transform.models._config import IGNORE_SYSTEM_KEYS +from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( + rdm_base_publication_model, +) +from cds_migrator_kit.transform.overdo import CdsOverdo + + +class TOTEMModel(CdsOverdo): + """Translation model for the TOTEM experiment.""" + + __query__ = "693__.e:TOTEM OR 980__.a:INTNOTETOTEMPUBL OR 980__.a:TOTEM_Papers OR 980__.a:TOTEM_Reports -980__:THESIS -980__:DELETED -980__:HIDDEN -980__:DUMMY" + + __ignore_keys__ = IGNORE_SYSTEM_KEYS + + _default_fields = { + "custom_fields": {}, + } + + +totem_model = TOTEMModel( + bases=(rdm_base_publication_model,), + entry_point_group="cds_migrator_kit.migrator.rules.totem", +) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index 87a8f70a..eebe10dc 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -879,3 +879,30 @@ def contact_person(self, key, value): } raise IgnoreKey("contributors") + + +@model.over("related_identifiers", "(^78502|^78002)") +@for_each_value +def related_works(self, key, value): + """Translates related identifiers.""" + description = StringValue(value.get("i")).parse().strip().lower() + recid = value.get("w") + report_number = value.get("r") + rel_ids = self.get("related_identifiers", []) + # See the decision log for 78502i/r/w + if "superseded by" == description: + relation_type = "isobsoletedby" + elif "supersedes" == description: + relation_type = "obsoletes" + else: + raise UnexpectedValue(f"Invalid relation type: {description}") + new_id = { + "identifier": recid, + "scheme": "cds", + "relation_type": {"id": relation_type}, + "resource_type": {"id": "other"}, + } + if new_id not in rel_ids: + return new_id + + raise IgnoreKey("related_identifiers") diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py index 6ec2e780..d4192cd4 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/ship.py @@ -30,29 +30,3 @@ def document_type(self, key, value): ]: raise UnexpectedValue(f"Invalid document type: {document_type}") raise IgnoreKey("document_type") - - -@model.over("related_identifiers", "(^78502|^78002)") -@for_each_value -def related_works(self, key, value): - """Translates related identifiers.""" - description = StringValue(value.get("i")).parse().strip().lower() - recid = value.get("w") - report_number = value.get("r") - rel_ids = self.get("related_identifiers", []) - if "superseded by" == description: - relation_type = "isobsoletedby" - elif "supersedes" == description: - relation_type = "obsoletes" - else: - raise UnexpectedValue(f"Invalid relation type: {description}") - new_id = { - "identifier": recid, - "scheme": "cds", - "relation_type": {"id": relation_type}, - "resource_type": {"id": "other"}, - } - if new_id not in rel_ids: - return new_id - - raise IgnoreKey("related_identifiers") diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index 28d64b40..4699e31f 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -689,6 +689,17 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "2ab3ab07-a634-4de8-8489-f99bc7fabec2" + totem: + data_dir: cds_migrator_kit/rdm/data/totem + plots: true + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/totem/dump + transform: + files_dump_dir: cds_migrator_kit/rdm/data/totem/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "" comments: faser-drafts: dir_path: /migration/faser-drafts/comments/ diff --git a/setup.cfg b/setup.cfg index ff4c615b..8de0cab1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -107,6 +107,7 @@ cds_migrator_kit.migrator.models = isolde = cds_migrator_kit.rdm.records.transform.models.isolde:isolde_model north_area = cds_migrator_kit.rdm.records.transform.models.north_area:north_area_model hie_isolde = cds_migrator_kit.rdm.records.transform.models.hie_isolde:hie_isolde_model + totem = cds_migrator_kit.rdm.records.transform.models.totem:totem_model cds_migrator_kit.migrator.rules.base = base = cds_migrator_kit.transform.xml_processing.rules.base @@ -247,6 +248,10 @@ cds_migrator_kit.migrator.rules.hie_isolde = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research +cds_migrator_kit.migrator.rules.totem = + base = cds_migrator_kit.transform.xml_processing.rules.base + base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base + publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research cds_migrator_kit.migrator.rules.people = people = cds_migrator_kit.rdm.users.transform.xml_processing.rules.people invenio_pidstore.minters =