From 3d7237f83880fdbc6bd39699c5c56aca2cba461a Mon Sep 17 00:00:00 2001 From: kpsherva Date: Fri, 16 Oct 2026 14:06:50 +0200 Subject: [PATCH 1/2] chore(comments): simplify comments migration --- scripts/export_collection_comments.py | 359 +++++++++++++++++++++++++ scripts/migrate_collection_comments.sh | 98 +++++++ 2 files changed, 457 insertions(+) create mode 100644 scripts/export_collection_comments.py create mode 100755 scripts/migrate_collection_comments.sh diff --git a/scripts/export_collection_comments.py b/scripts/export_collection_comments.py new file mode 100644 index 00000000..db294f9b --- /dev/null +++ b/scripts/export_collection_comments.py @@ -0,0 +1,359 @@ +""" +One-shot export of a collection's legacy comments (run on cds-migration-01). + +Consolidates, in this order: + 1. dump_comments_to_migrate.py -> comments_metadata.json + + active_commentors.json + + missing_commentors_from_ldap.json + 2. copying people.csv -> /users/people.csv + 3. copy_comments_attached_files.py -> //... + +Usage (Python 2 / ipython of the legacy Invenio, so no f-strings here): + + ipython ./scripts/export_collection_comments.py -- \ + --env dev --collection antares_restr --queries-file antares_queries.txt + + # or pass queries inline (repeatable) + ipython ./scripts/export_collection_comments.py -- \ + --env dev --collection antares_restr --query '980__a:PRIVANTARES ...' + +The queries file has one legacy search query per line (blank lines and lines +starting with '#' are ignored). Use the same queries as for the collection +migration. + +Output (everything under /eos/media/cds/cds-rdm//migration//comments/): + comments_metadata.json + // + users/people.csv, users/active_commentors.json, + users/missing_commentors_from_ldap.json + +Afterwards, on the migration pod, run scripts/migrate_collection_comments.sh. + +The script is safe to re-run: files are overwritten and already copied +attachment folders are skipped. +""" + +import argparse +import io +import json +import os +import shutil +import sys + +from invenio.bibcirculation_cern_ldap import get_user_info_from_ldap +from invenio.dbquery import run_sql +from invenio.search_engine import search_pattern +from invenio.webcomment_dblayer import get_comment_to_bibdoc_relations + +EOS_ROOT = "/eos/media/cds/cds-rdm/{env}/migration" +LEGACY_COMMENTS_FILES_DIR = "/opt/cdsweb/var/data/comments" + + +# -------------------------------------------------------------------------- +# Legacy comments extraction +# -------------------------------------------------------------------------- +def get_comments_from_legacy(recid, display_order="cmt.date_creation ASC"): + """Fetch all comments of a record. + + Mirrors invenio.webcomment.query_retrieve_comments_or_remarks. + """ + query = """SELECT user.nickname, + user.email, + user.note, + user.last_login, + cmt.id_user, + DATE_FORMAT(cmt.date_creation, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s') as created, + cmt.body, + cmt.status, + cmt.nb_abuse_reports, + cmt.id, + cmt.round_name, + cmt.restriction, + %(reply_to_column)s as reply_to_id, + cmt.body_format + FROM cmtRECORDCOMMENT cmt LEFT JOIN user ON + user.id=cmt.id_user + WHERE cmt.id_bibrec=%%s + ORDER BY %(display_order)s + """ % { + "display_order": display_order, + "reply_to_column": recid > 0 + and "cmt.in_reply_to_id_cmtRECORDCOMMENT" + or "cmt.in_reply_to_id_bskRECORDCOMMENT", + } + return run_sql(query, (recid,), with_dict=True) + + +def flatten_replies(comments_list): + """Return top-level comments with all (nested) replies flattened to one level.""" + comments_by_id = {c["id"]: dict(c, replies=[]) for c in comments_list} + top_level_comments = [] + + for c in comments_list: + parent = c.get("reply_to_id") + if parent is None or parent not in comments_by_id: + top_level_comments.append(comments_by_id[c["id"]]) + else: + comments_by_id[parent]["replies"].append(comments_by_id[c["id"]]) + + for comment in top_level_comments: + flat = [] + queue = list(comment["replies"]) + while queue: + reply = queue.pop(0) + flat.append(reply) + queue.extend(reply["replies"]) + for r in flat: + r["replies"] = [] + flat.sort(key=lambda x: x["created"]) + comment["replies"] = flat + + return top_level_comments + + +def find_recids_with_comments(queries): + """Return the recids matching any of the queries that have comments.""" + recids = [] + print("Querying already migrated records...") + for i, collection_query in enumerate(queries): + recs = search_pattern(p=collection_query) + print( + "({} / {}) Found {} records for query `{}`".format( + i + 1, len(queries), len(recs), collection_query + ) + ) + recids.extend(recs) + + recids = sorted(set(recids)) + if not recids: + return [] + + print("Filtering records with comments...") + rows = run_sql( + "SELECT id_bibrec " + "FROM cmtRECORDCOMMENT " + "WHERE id_bibrec IN ({}) " + "GROUP BY id_bibrec " + "HAVING COUNT(*) > 0".format(",".join(str(recid) for recid in recids)), + ) + recids_with_comments = [row[0] for row in rows] + print( + "Found {} records with comments out of {} records".format( + len(recids_with_comments), len(recids) + ) + ) + return recids_with_comments + + +def _sanitize(body): + return body.replace("\xc2\xa0", " ").replace("\n", "").strip() + + +def dump_comments(recids): + """Build the comments metadata (see CommentsRunner) and the commenters set.""" + comments_metadata = {} + users_metadata = set() + + for i, recid in enumerate(recids): + print("({}/{}) Processing comments for record<{}>".format(i + 1, len(recids), recid)) + comments = get_comments_from_legacy(recid) + + if not comments: + print("No comments found for record<{}>. Skipping...".format(recid)) + continue + + # skip records whose comments are all deleted/spam, e.g. + # https://cds.cern.ch/record/1367848/comments?ln=en + if not any(comment["status"] in ["ok", "ap"] for comment in comments): + print( + "No 'ok'/'ap' comment found for record<{}>. Skipping...".format(recid) + ) + continue + + print("Found `{}` comment(s) for record<{}>".format(len(comments), recid)) + comments_metadata[recid] = [] + + # comments attached to a record's file (and file version); not to be + # confused with files attached to the comments themselves + relations = get_comment_to_bibdoc_relations(recid) + file_relations = { + r["id_comment"]: {"file_id": r["id_bibdoc"], "version": r["version"]} + for r in relations + } + + for comment in comments: + users_metadata.add( + ( + comment["id_user"], + comment["email"], + comment["nickname"], + comment["note"], + comment["last_login"], + ) + ) + + for comment in flatten_replies(comments): + comments_metadata[recid].append( + { + "comment_id": comment["id"], + "content": _sanitize(comment["body"]), + "status": comment.get("status"), + "user_email": comment.get("email"), + "created_at": comment.get("created"), + "file_relation": file_relations.get(comment["id"], {}), + "replies": [ + { + "comment_id": reply["id"], + "content": _sanitize(reply["body"]), + "status": reply.get("status"), + "user_email": reply.get("email"), + "created_at": reply.get("created"), + "reply_to_id": reply.get("reply_to_id"), + "file_relation": file_relations.get(reply["id"], {}), + } + for reply in comment["replies"] + ], + } + ) + + return comments_metadata, users_metadata + + +# -------------------------------------------------------------------------- +# Commenters (from dump_users.py in the production_scripts repository: +# https://gitlab.cern.ch/cds-team/production_scripts/-/blob/master/cds-rdm/migration/dump_users.py) +# -------------------------------------------------------------------------- +def _ldap_field(ldap_user, field): + try: + return ldap_user[field][0] + except Exception: + return None + + +def dump_users(users, users_filepath, missing_users_filepath): + """Write active_commentors.json and missing_commentors_from_ldap.json.""" + valid_users, missing_users = [], [] + + for rec in users: + email = rec[1] + ldap_user = get_user_info_from_ldap(email=email) + uid_number = _ldap_field(ldap_user, "uidNumber") + record = { + "id": rec[0], + "email": rec[1], + "displayname": rec[2], + "active": rec[3], + } + if uid_number: + record["uid"] = uid_number + department = _ldap_field(ldap_user, "department") + if department: + record["department"] = department + else: + print("No department for {}".format(email)) + valid_users.append(record) + else: + missing_users.append(record) + + with open(users_filepath, "w") as fp: + json.dump(valid_users, fp, indent=2) + + # always written (possibly empty), so the pod step never reads a stale file + print( + "Missing users found {}. Creating {} file...".format( + len(missing_users), missing_users_filepath + ) + ) + with open(missing_users_filepath, "w") as fp: + json.dump(missing_users, fp, indent=2) + + +# -------------------------------------------------------------------------- +# Files +# -------------------------------------------------------------------------- +def copy_people_csv(env, users_dir): + source = os.path.join(EOS_ROOT.format(env=env), "users", "people.csv") + if not os.path.exists(source): + print("WARNING: {} not found, copy people.csv manually!".format(source)) + return + shutil.copy(source, os.path.join(users_dir, "people.csv")) + print("Copied people.csv to {}".format(users_dir)) + + +def copy_attached_files(recids, comments_dir): + """Copy the files attached to comments, one folder per record.""" + copied = 0 + for recid in recids: + source = os.path.join(LEGACY_COMMENTS_FILES_DIR, str(recid)) + destination = os.path.join(comments_dir, str(recid)) + if not os.path.isdir(source): + continue # no attachments for this record + if os.path.exists(destination): + print("Attachments of record<{}> already copied, skipping".format(recid)) + continue + shutil.copytree(source, destination) + copied += 1 + print("Copied {} comments directory to {}".format(recid, destination)) + print("Copied attachments of {} record(s)".format(copied)) + + +# -------------------------------------------------------------------------- +# Main +# -------------------------------------------------------------------------- +def read_queries(args): + queries = list(args.query or []) + if args.queries_file: + with io.open(args.queries_file, encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if line and not line.startswith("#"): + queries.append(line.encode("utf-8") if str is bytes else line) + if not queries: + sys.exit("No queries given: use --query and/or --queries-file") + return queries + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n")[1]) + parser.add_argument("--env", required=True, help="dev / qa / prod") + parser.add_argument("--collection", required=True, help="collection folder name") + parser.add_argument("--query", action="append", help="legacy search query") + parser.add_argument("--queries-file", help="file with one query per line") + args = parser.parse_args() + + queries = read_queries(args) + comments_dir = os.path.join( + EOS_ROOT.format(env=args.env), args.collection, "comments" + ) + users_dir = os.path.join(comments_dir, "users") + for path in (comments_dir, users_dir): + if not os.path.exists(path): + os.makedirs(path) + + recids = find_recids_with_comments(queries) + comments_metadata, users = dump_comments(recids) + if not comments_metadata: + sys.exit("No comments to migrate for this collection, nothing written.") + + comments_filepath = os.path.join(comments_dir, "comments_metadata.json") + print("Creating {} file...".format(comments_filepath)) + with open(comments_filepath, "w") as fp: + json.dump(comments_metadata, fp) + + dump_users( + users, + os.path.join(users_dir, "active_commentors.json"), + os.path.join(users_dir, "missing_commentors_from_ldap.json"), + ) + copy_people_csv(args.env, users_dir) + copy_attached_files(comments_metadata.keys(), comments_dir) + + print("\nDone. Next, on the migration pod:") + print( + " ./scripts/migrate_collection_comments.sh --collection {}".format( + args.collection + ) + ) + + +main() diff --git a/scripts/migrate_collection_comments.sh b/scripts/migrate_collection_comments.sh new file mode 100755 index 00000000..b3f75422 --- /dev/null +++ b/scripts/migrate_collection_comments.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# +# One-shot comments migration for a collection (run on the migration pod). +# +# Consolidates, in this order: +# 1. pre-flight checks (files from export_collection_comments.py are there, +# the collection is configured in streams.yaml) +# 2. invenio migration comments commenters-run (dry run, then real run) +# 3. invenio migration comments run (dry run, then real run) +# +# Usage: +# ./scripts/migrate_collection_comments.sh --collection antares_restr +# ./scripts/migrate_collection_comments.sh --collection antares_restr --yes +# ./scripts/migrate_collection_comments.sh --collection antares_restr --dry-run-only +# +# The input paths are NOT passed on the command line: the CLI reads them from the +# `comments:` section of streams.yaml (`dir_path`, plus `/users`). +# +# Commenters are indexed by the loader right when they are created, so no manual +# re-indexing is needed. Both runners can be re-run: already created users and +# comments are skipped. + +set -euo pipefail + +COLLECTION="" +ASSUME_YES=false +DRY_RUN_ONLY=false +STREAMS_FILE="${CDS_MIGRATOR_KIT_STREAM_CONFIG:-}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --collection) COLLECTION="$2"; shift 2 ;; + --yes|-y) ASSUME_YES=true; shift ;; + --dry-run-only) DRY_RUN_ONLY=true; shift ;; + --streams-file) STREAMS_FILE="$2"; shift 2 ;; + -h|--help) sed -n '2,19p' "$0"; exit 0 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$COLLECTION" ]] || { echo "--collection is required" >&2; exit 2; } + +# --- 1. pre-flight checks --------------------------------------------------- +# dir_path of the collection, from the same streams.yaml the runners use. +DIR_PATH="$(python - "$COLLECTION" "$STREAMS_FILE" <<'EOF' +import sys +import yaml +from flask import current_app +from invenio_app.factory import create_app + +collection, streams_file = sys.argv[1], sys.argv[2] +if not streams_file: + with create_app().app_context(): + streams_file = current_app.config["CDS_MIGRATOR_KIT_STREAM_CONFIG"] +with open(streams_file) as fp: + config = yaml.safe_load(fp) +try: + print(config["comments"][collection]["dir_path"]) +except KeyError: + sys.exit(f"collection '{collection}' has no entry under `comments:` in {streams_file}") +EOF +)" + +echo "Comments directory: $DIR_PATH" +missing=0 +for f in "comments_metadata.json" "users/missing_commentors_from_ldap.json"; do + if [[ ! -f "$DIR_PATH/$f" ]]; then + echo "MISSING: $DIR_PATH/$f (run scripts/export_collection_comments.py first)" >&2 + missing=1 + fi +done +[[ -f "$DIR_PATH/users/people.csv" ]] || echo "WARNING: $DIR_PATH/users/people.csv not found" +[[ $missing -eq 0 ]] || exit 1 + +confirm() { + $ASSUME_YES && return 0 + read -r -p "$1 [y/N] " answer + [[ "$answer" =~ ^[Yy]$ ]] +} + +# The runners catch and log their own errors, so a dry run exit code can't be +# trusted to tell success from failure: check the logs before confirming. +run_step() { + local name="$1" cmd="$2" + echo; echo "=== $name: dry run ===" + invenio migration comments "$cmd" --collection "$COLLECTION" --dry-run + $DRY_RUN_ONLY && return 0 + confirm "Dry run of '$name' finished. Check the logs, then run it for real?" \ + || { echo "Stopped before '$name'."; exit 1; } + echo "=== $name: real run ===" + invenio migration comments "$cmd" --collection "$COLLECTION" +} + +# --- 2. commenters, 3. comments -------------------------------------------- +run_step "commenters" commenters-run +run_step "comments" run + +echo; echo "Done. Re-run is safe: existing users and comments are skipped." From b735994e602a3c19364906bc17a6d660fbc141a1 Mon Sep 17 00:00:00 2001 From: kpsherva Date: Fri, 23 Oct 2026 17:59:57 +0200 Subject: [PATCH 2/2] refactor(global): remove manual steps from migration * consolidates substeps (check readme and migrating-a-collection.md) # Conflicts: # .gitignore # Dockerfile # cds_migrator_kit/rdm/streams.yaml # cds_migrator_kit/rdm/streams_done.yaml # cds_migrator_kit/rdm/streams_shelved.yaml --- .gitignore | 5 +- cds_migrator_kit/__init__.py | 1 - cds_migrator_kit/rdm/README.md | 389 +++++---------- cds_migrator_kit/rdm/cli.py | 258 +++++++++- cds_migrator_kit/rdm/collection_conf.py | 171 +++++++ cds_migrator_kit/rdm/comments/runner.py | 28 +- cds_migrator_kit/rdm/data/cms-notes/.gitkeep | 0 .../rdm/data/summer_student_reports/.gitkeep | 0 .../duplicated_pids.json | 20 - cds_migrator_kit/rdm/migration_config.py | 17 + cds_migrator_kit/rdm/readme_models.md | 63 +++ cds_migrator_kit/rdm/streams.yaml | 200 +------- cds_migrator_kit/rdm/streams_done.yaml | 60 --- cds_migrator_kit/rdm/streams_shelved.yaml | 1 + cds_migrator_kit/reports/log.py | 8 +- cds_migrator_kit/reports/utils.py | 1 + cds_migrator_kit/runner/runner.py | 17 +- cds_migrator_kit/transform/dumper.py | 1 + .../xml_processing/quality/parsers.py | 1 + cds_migrator_kit/users/load.py | 5 +- cds_migrator_kit/users/transform.py | 1 + migration_cli/collection_migration.py | 458 ++++++++++++++++++ migration_cli/collections/_example.yaml | 43 ++ migration_cli/collections/ab_dep.yaml | 10 + migration_cli/collections/ac_dep.yaml | 10 + migration_cli/collections/antares.yaml | 9 + migration_cli/collections/antares_restr.yaml | 9 + migration_cli/collections/ar_dep.yaml | 10 + migration_cli/collections/at.yaml | 9 + migration_cli/collections/dirac.yaml | 9 + migration_cli/collections/drdc_comm.yaml | 7 + migration_cli/collections/eec_comm.yaml | 7 + migration_cli/collections/emc_comm.yaml | 7 + migration_cli/collections/est_dep.yaml | 8 + migration_cli/collections/imxgam.yaml | 11 + migration_cli/collections/isc_comm.yaml | 7 + migration_cli/collections/isolde.yaml | 10 + migration_cli/collections/isr_dep.yaml | 10 + migration_cli/collections/isrc_comm.yaml | 7 + migration_cli/collections/istc_comm.yaml | 7 + migration_cli/collections/lep_dep.yaml | 10 + migration_cli/collections/lepc_comm.yaml | 7 + migration_cli/collections/mps_div.yaml | 10 + migration_cli/collections/msc_div.yaml | 10 + migration_cli/collections/mt_dep.yaml | 8 + migration_cli/collections/na61.yaml | 9 + migration_cli/collections/na62.yaml | 9 + migration_cli/collections/na63.yaml | 9 + migration_cli/collections/na64.yaml | 9 + migration_cli/collections/na65.yaml | 9 + migration_cli/collections/na66.yaml | 9 + migration_cli/collections/nprc_comm.yaml | 7 + migration_cli/collections/nsc_comm.yaml | 7 + migration_cli/collections/phi3_comm.yaml | 7 + migration_cli/collections/phi_comm.yaml | 7 + migration_cli/collections/ps_dep.yaml | 10 + migration_cli/collections/psc_comm.yaml | 7 + migration_cli/collections/pscc_comm.yaml | 7 + migration_cli/collections/re29.yaml | 10 + migration_cli/collections/sb_dep.yaml | 8 + migration_cli/collections/sc_dep.yaml | 10 + migration_cli/collections/scc_comm.yaml | 7 + migration_cli/collections/scp_adv.yaml | 7 + migration_cli/collections/si_dep.yaml | 10 + migration_cli/collections/sl_dep.yaml | 10 + migration_cli/collections/sps_dep.yaml | 10 + migration_cli/collections/spsc_comm.yaml | 7 + migration_cli/collections/spslc_comm.yaml | 7 + migration_cli/collections/tcc_comm.yaml | 7 + migration_cli/collections/ts_dep.yaml | 8 + migration_cli/collections/ua2.yaml | 10 + migration_cli/collections/ua4.yaml | 9 + migration_cli/collections/ua5.yaml | 9 + migration_cli/collections/ua8.yaml | 9 + migration_cli/migrating-a-collection.md | 153 ++++++ .../data => migration_cli/rdm-data}/.gitkeep | 0 migration_cli/step_by_step_documentation.md | 295 +++++++++++ .../analyze_collection_migration.py | 28 +- .../subscripts}/copy_collection_files.py | 55 ++- .../subscripts}/export_collection_comments.py | 36 +- scripts/copy_comments_attached_files.py | 52 -- scripts/dump_comments_to_migrate.py | 382 --------------- scripts/migrate_collection_comments.sh | 98 ---- tests/cds-rdm/helpers.py | 6 +- 84 files changed, 2128 insertions(+), 1161 deletions(-) create mode 100644 cds_migrator_kit/rdm/collection_conf.py delete mode 100644 cds_migrator_kit/rdm/data/cms-notes/.gitkeep delete mode 100644 cds_migrator_kit/rdm/data/summer_student_reports/.gitkeep delete mode 100644 cds_migrator_kit/rdm/data/summer_student_reports/duplicated_pids.json create mode 100644 cds_migrator_kit/rdm/readme_models.md create mode 100644 migration_cli/collection_migration.py create mode 100644 migration_cli/collections/_example.yaml create mode 100644 migration_cli/collections/ab_dep.yaml create mode 100644 migration_cli/collections/ac_dep.yaml create mode 100644 migration_cli/collections/antares.yaml create mode 100644 migration_cli/collections/antares_restr.yaml create mode 100644 migration_cli/collections/ar_dep.yaml create mode 100644 migration_cli/collections/at.yaml create mode 100644 migration_cli/collections/dirac.yaml create mode 100644 migration_cli/collections/drdc_comm.yaml create mode 100644 migration_cli/collections/eec_comm.yaml create mode 100644 migration_cli/collections/emc_comm.yaml create mode 100644 migration_cli/collections/est_dep.yaml create mode 100644 migration_cli/collections/imxgam.yaml create mode 100644 migration_cli/collections/isc_comm.yaml create mode 100644 migration_cli/collections/isolde.yaml create mode 100644 migration_cli/collections/isr_dep.yaml create mode 100644 migration_cli/collections/isrc_comm.yaml create mode 100644 migration_cli/collections/istc_comm.yaml create mode 100644 migration_cli/collections/lep_dep.yaml create mode 100644 migration_cli/collections/lepc_comm.yaml create mode 100644 migration_cli/collections/mps_div.yaml create mode 100644 migration_cli/collections/msc_div.yaml create mode 100644 migration_cli/collections/mt_dep.yaml create mode 100644 migration_cli/collections/na61.yaml create mode 100644 migration_cli/collections/na62.yaml create mode 100644 migration_cli/collections/na63.yaml create mode 100644 migration_cli/collections/na64.yaml create mode 100644 migration_cli/collections/na65.yaml create mode 100644 migration_cli/collections/na66.yaml create mode 100644 migration_cli/collections/nprc_comm.yaml create mode 100644 migration_cli/collections/nsc_comm.yaml create mode 100644 migration_cli/collections/phi3_comm.yaml create mode 100644 migration_cli/collections/phi_comm.yaml create mode 100644 migration_cli/collections/ps_dep.yaml create mode 100644 migration_cli/collections/psc_comm.yaml create mode 100644 migration_cli/collections/pscc_comm.yaml create mode 100644 migration_cli/collections/re29.yaml create mode 100644 migration_cli/collections/sb_dep.yaml create mode 100644 migration_cli/collections/sc_dep.yaml create mode 100644 migration_cli/collections/scc_comm.yaml create mode 100644 migration_cli/collections/scp_adv.yaml create mode 100644 migration_cli/collections/si_dep.yaml create mode 100644 migration_cli/collections/sl_dep.yaml create mode 100644 migration_cli/collections/sps_dep.yaml create mode 100644 migration_cli/collections/spsc_comm.yaml create mode 100644 migration_cli/collections/spslc_comm.yaml create mode 100644 migration_cli/collections/tcc_comm.yaml create mode 100644 migration_cli/collections/ts_dep.yaml create mode 100644 migration_cli/collections/ua2.yaml create mode 100644 migration_cli/collections/ua4.yaml create mode 100644 migration_cli/collections/ua5.yaml create mode 100644 migration_cli/collections/ua8.yaml create mode 100644 migration_cli/migrating-a-collection.md rename {cds_migrator_kit/rdm/data => migration_cli/rdm-data}/.gitkeep (100%) create mode 100644 migration_cli/step_by_step_documentation.md rename {scripts => migration_cli/subscripts}/analyze_collection_migration.py (93%) rename {scripts => migration_cli/subscripts}/copy_collection_files.py (70%) rename {scripts => migration_cli/subscripts}/export_collection_comments.py (91%) delete mode 100644 scripts/copy_comments_attached_files.py delete mode 100644 scripts/dump_comments_to_migrate.py delete mode 100755 scripts/migrate_collection_comments.sh diff --git a/.gitignore b/.gitignore index 9f9f49a9..afc438a5 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,7 @@ target/ snapshots/ **.DS_Store -**/cds_migrator_kit/rdm/data/ +# local migration data (dumps, files, people.csv...): real data, never commit it. +# Only the rdm-data folder itself is kept, through its .gitkeep +migration_cli/rdm-data/* +!migration_cli/rdm-data/.gitkeep diff --git a/cds_migrator_kit/__init__.py b/cds_migrator_kit/__init__.py index d0a8de1b..be8a211c 100644 --- a/cds_migrator_kit/__init__.py +++ b/cds_migrator_kit/__init__.py @@ -8,7 +8,6 @@ """Migration tool kit from old Invenio to new Invenio.""" - from .ext import CdsMigratorKit __all__ = ("__version__", "CdsMigratorKit") diff --git a/cds_migrator_kit/rdm/README.md b/cds_migrator_kit/rdm/README.md index 5c23c6bc..22b73f97 100644 --- a/cds_migrator_kit/rdm/README.md +++ b/cds_migrator_kit/rdm/README.md @@ -1,351 +1,196 @@ -# Migration model manual +# Migrating a collection locally (example: `antares`) -## Dump users +How to run a collection migration on your own machine, isolated from EOS and OpenShift: the migration data lives in a local folder, and the legacy system is only used to produce the dumps. -! Attention If you need to dump the users from legacy DB or you need to process the people collection +Other documents: -https://gitlab.cern.ch/cds-team/production_scripts/-/blob/master/cds-rdm/migration/dump_users.py?ref_type=heads +- migration on the test environments: `migration_cli/migrating-a-collection.md` +- each stage in detail: `migration_cli/step_by_step_documentation.md` +- writing the transformation models: `cds_migrator_kit/rdm/readme_models.md` +## What is different locally +With `CDS_MIGRATOR_KIT_ENV` set to `local` the loader does not read the legacy files: **every file is replaced by a dummy PDF**, and the checksum and access-grant verifications are skipped. So you only need the records dump (and the comments metadata); there is no need to copy the legacy files. -## Dump a subset of records on legacy +## 0. Prerequisites -on webnode: `cds-migration-01` +- the services running (PostgreSQL, OpenSearch, RabbitMQ, Redis) and a Celery worker +- the package installed: `pip install -e ".[rdm]"` +- `people.csv` of the environment, see step 2 -```bash -inveniomigrator dump records -q '980:INTNOTECMSPUBL 980:NOTE -980:DELETED' --file-prefix cms-notes --chunk-size=1000 +### Initialise the instance (first time only) +Run in the cds-rdm environment, on empty database and indices (the commands are those of `reinit-instance.sh`): + +```shell +invenio db init +invenio index init +invenio rdm-records custom-fields init +invenio communities custom-fields init +invenio rdm-records fixtures # vocabularies, the CERN Research community (the common community id used in the confs), ... +invenio rdm fixtures # static pages +invenio rdm rebuild-all-indices ``` -## Define transforming models and rules resolution +`invenio rdm-records fixtures` loads everything declared in cds-rdm's `app_data/` (`vocabularies.yaml`: subjects, affiliations, funders, awards, experiments, departments, committees, accelerators, beams, programmes, resource types, community types, ...; and `communities.yaml`). When the files in `app_data/` change later, update the existing entries with `invenio rdm-records add-to-fixture ` (`programmes`, `subjects`, `accelerators`, `awards`, `experiments`, `resourcetypes`, `communitytypes`). -You can adapt XML processing to different subsets of records by implementing different data models for each subset (f.e. collection). -Let's take CMS notes as an example: +Then populate the vocabularies that are not shipped as fixtures: the ROR affiliations and the ORCID names. They are the Celery task `process_datastream` of invenio-vocabularies, which you can call from `invenio shell`, using the configuration of the corresponding jobs. The Celery worker must be running: the tasks only read the source and hand the entries over to the worker. ```python +# invenio shell +from invenio_vocabularies.jobs import ImportORCIDJob, ProcessRORAffiliationsJob +from invenio_vocabularies.services.tasks import process_datastream -class CMSNote(CdsOverdo): - """Translation Index for CDS Books.""" - - __query__ = ( - '980__:INTNOTECMSPUBL 980__:NOTE' - ) - - __model_ignore_keys__ = {} - - _default_fields = {} - - - -model = CMSNote(bases=(), - entry_point_group="cds_rdm.migrator.rules" - ) +# ROR affiliations +process_datastream.delay(**ProcessRORAffiliationsJob.build_task_arguments(None)) +# ORCID names +process_datastream.delay(**ImportORCIDJob._build_task_arguments(None)) ``` -**query** - defines the MARC fields to which specific record should match. Attention: It does not recognise regexes that are used to specify the collection query in the admin interface of legacy CDS. - -\***\*model_ignore_keys\*\*** - set of keys to be ignored for this data model - fields will not be migrated - -**bases** - by defining bases of models you can specify a parent model which fits all the subsets of records (f.e. 245 - title field MARC to JSON translation could be the same for all the models) - -**entry_point_group** - reference to where the model should lookup for the set of the MARC translation rules, see the entrypoints below. +- `ProcessRORAffiliationsJob` downloads the ROR dump into the affiliations vocabulary (the fixtures only load the subset in `app_data/vocabularies/affiliations_ror.yaml`). +- `ImportORCIDJob` reads the ORCID summaries from the ORCID S3 bucket, and needs `VOCABULARIES_ORCID_ACCESS_KEY` and `VOCABULARIES_ORCID_SECRET_KEY` in your configuration. `VOCABULARIES_ORCID_SYNC_SINCE` limits how far back it goes. +- `.delay(...)` queues the task and returns at once, follow it in the Celery worker log. To run the reading in the shell itself, call `process_datastream(**...)` without `.delay`. -After defining your model and set of rules,you have to register them in the entrypoints of your application, in setup.cfg: - -```editorconfig - -[options.entry_points] -cds_migrator_kit.migrator.models = - cms_note = cds_rdm.migration.transform.models.note:model -cds_migrator_kit.migrator.rules = - base = cds_rdm.migration.transform.xml_processing.rules - -``` +### Save a "clean slate" snapshot -## Full migration workflow of one collection - -### To visualise the errors (locally): +The instance takes long to initialise (vocabularies, ROR, ORCID), and every migration run leaves records behind. Right after the initialisation, and before any migration, dump the clean state with `scripts/snapshot.sh` (PostgreSQL and OpenSearch together, in `snapshots//`): ```shell -gunicorn -b :8080 --timeout 120 --graceful-timeout 60 cds_migrator_kit.app:app --reload --log-level debug --capture-output --access-logfile '-' --error-logfile '-' +scripts/snapshot.sh dump clean-slate ``` -### Legacy - -This is the recipe on how to dump the metadata and files. For local tests it is not necessary to do it often, especially for files which are static. -It makes sense to dump metadata if any changes are applied on legacy +and bring the instance back to it in a few minutes whenever you want to rerun a migration from scratch: ```shell -ssh cds-migration-01 # inveniomigrator tool installed here -cd /tmp/current-collection-dump -inveniomigrator dump records -q '037__:CERN-STUDENTS-Note-* -980:DELETED' --file-prefix summer-studends-notes --latest-only --chunk-size=1000 - -cd /eos/media/cds/cds-rdm/dev/migration/summer-student-notes/dump -kinit cdsrdmeosdev -cp /tmp/current-collection-dump/* . - - -# copy over all files to temporary location -ipython -# run script available in /scripts/copy_collection_files.py -# destination_prefix = "/eos/media/cds/cds-rdm/dev/migration/summer-student-notes/files" -# working_dir = "/eos/media/cds/cds-rdm/dev/migration/summer-student-notes" -# json_dump_dir = "/eos/media/cds/cds-rdm/dev/migration/summer-student-notes/dump" +scripts/snapshot.sh restore clean-slate # asks before destroying the current data, FORCE=1 skips it ``` -Identify and dump duplicated records i.e. records with same title, description and file -checksums (excluding deleted files) on EOS. If duplicates found, then we keep the recid -that has restricted files or if not any, then the first one from the list. - -In a python shell run the script `scripts/dump_legacy_recids_to_redirect.py`. - -#### How to mount eos locally on MAC (to copy over the dumps and files) +The script expects a running PostgreSQL container and the cds-rdm virtualenv in `../cds-rdm/.venv` (`CDS_RDM_VENV_ACTIVATE` to change it), see its header. `scripts/postgres-init.sh` and `scripts/opensearch-init.sh` do each half on its own. -1. Go to Finder icon on your dock -2. right click to get the contextual menu -3. choose connect to server -4. type `https://cernbox.cern.ch/cernbox/webdav/eos/media/cds/cds-rdm/dev/migration` -5. click connect -6. use eos account dev credentials +**Keep the snapshot up to date.** It stores the data model as it was when it was taken, so refresh it after upgrading cds-rdm or invenio: -### Openshift migration pod +1. restore the snapshot, then upgrade the database schema: `invenio alembic upgrade` +2. update the OpenSearch indices to the new mappings: the restore recreates the indices from the code with `invenio index init`, but an index that already exists is changed with `invenio index update `; then reindex with `invenio rdm rebuild-all-indices` +3. run `invenio rdm-records custom-fields init` and `invenio communities custom-fields init` if custom fields were added, and `invenio rdm-records add-to-fixture ...` for changed fixtures +4. dump it again: `scripts/snapshot.sh dump clean-slate` (same name overwrites it, or use a dated name) -#### Collect and dump affiliation mapping +## 1. Local data folder -In order to collect all affiliations from the collection dump folder run the following -command pointing to the `cds_migrator_kit.rdm.data.summer_student_reports.dump` -folder: +The local "root" of the migration data is the folder `migration_cli/rdm-data/` of this repository (the default of the `local` environment; relative paths are relative to the repository root, whatever the directory you run `invenio` from). The commands below are run from the repository root. The working directory of a collection is `//`: ``` -invenio migration affiliations run --filepath /path/to/cds_migrator_kit/rdm/migration/data/summer_student_reports/dump -``` - -This will collect and check each affiliation against the ROR organization API, and store them in the `cds_rdm.legacy.models.CDSMigrationAffiliationMapping` table. - -The model is then used during record migration to normalize the affiliation content following the below principles -to map the legacy input to a normalized value: - -1. If curated affiliation that might or not have a ROR ID exists then this value is used. -2. A ROR exact match -3. A ROR not exact match with a level of confidence of >= 90%. This will also flag the - record for further curation to validate the value. -4. The legacy affiliation value, and flag the record. - -#### Community id dump - -Dump the community id to migration `streams.yaml`(if the slug exists then it will read and dump the id only): - -```shell -invenio migration community dump --slug 'sspn' --title 'Summer Student Project Notes' --filepath /path/to/cds_migrator_kit/rdm/migration/streams.yaml +/migration_cli/rdm-data/ <- the root (default of the local environment) + users/people.csv shared by all collections + antares/ <- the working directory of the collection + dump/ records dump + comments/ optional, comments of the collection ``` -#### Missing Users files - -Make sure to add the user files required to be able to run the migration. The files can be found in `/eos/media/cds/cds-rdm/dev/migration/users/`. They should be stored in path indicated in the streams.yaml, the value that corresponds to the `record.transform.missing_users` key. - -#### Records migration - -Run the below command to migrate records in the created community from before: - ```shell -invenio migration run +mkdir -p migration_cli/rdm-data/users migration_cli/rdm-data/antares ``` -#### EP approval records +`people.csv` is the people authority export of the environment. Copy it to `migration_cli/rdm-data/users/people.csv` from the EOS `users` folder, or recreate it as described in `migration_cli/migrating-a-collection.md`. -EP approval records are detected automatically per-record (via the `9031_:EPPHAPP` history popped during transform) and split into a public/restricted pair by the loader - no separate flag or stream is needed, they can be migrated in the same run as regular records. +## 2. Configuration -1. **Dump** them filtered by `9031_:EPPHAPP`, for example (FASER): +The configuration of a collection is the file `migration_cli/collections/antares.yaml` of the repository, the same for every environment (all the options in `migration_cli/collections/_example.yaml`): -```bash -inveniomigrator dump records -q '980__a:ARTICLE or 980__a:PREPRINT and 693:"FASER" not 980:CONFERENCEPAPER not 591__b:"Draft" 9031_:EPPHAPP -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY' --file-prefix faser-papers-cern-ep --chunk-size=1000 +```yaml +name: antares +queries: + - '' +flags: + plots: true +communities: + - cern-research + - antares ``` -2. Configure a collection entry in `streams.yaml` pointing at that dump. - -3. **Migrate** normally (dry run first): +The communities are given by slug and are resolved to ids by the migration, so they must exist in your local instance. `cern-research` is created by the fixtures (step 0). Create the collection community, in `invenio shell`: -```shell -invenio migration run --collection faser-ep --dry-run -invenio migration run --collection faser-ep +```python +from invenio_access.permissions import system_identity +from invenio_communities.proxies import current_communities + +service = current_communities.service +try: + service.read(system_identity, "antares") +except Exception: # not created yet + service.create( + system_identity, + {"slug": "antares", "metadata": {"title": "Antares"}, "access": {"visibility": "public"}}, + ) ``` -#### Comments migration - -Configure the collection under `comments` in `streams.yaml` (`dir_path`, `reviewers`). Paths and reviewers are loaded from there when you pass `--collection`. +## 3. Get the dumps from the legacy system -1. Dump comments for the collection and generate commenter metadata: +The legacy tools exist only on `cds-migration-01`. Run the script there with a scratch folder as root (not on EOS), then download the result: ```shell # on cds-migration-01 -ipython ./scripts/dump_comments_to_migrate.py -``` +S=/opt/cds-migrator-kit/migration_cli # a checkout that has the configuration of the collection +python $S/collection_migration.py dump records --collection antares --env local --root /tmp/antares-root +ipython $S/collection_migration.py -- dump comments --collection antares --env local --root /tmp/antares-root # optional +python $S/collection_migration.py prepare --collection antares --env local --root /tmp/antares-root # optional -Output: `comments_metadata.json`. The script also generates `missing_commentors_from_ldap.json`. +# on your machine +rsync -av cds-migration-01:/tmp/antares-root/antares/dump/ migration_cli/rdm-data/antares/dump/ +rsync -av cds-migration-01:/tmp/antares-root/antares/comments/ migration_cli/rdm-data/antares/comments/ # optional +rsync -av cds-migration-01:/tmp/antares-root/antares/{duplicated_pids,ep_report_number_dups}.json migration_cli/rdm-data/antares/ # optional, after prepare +``` -**Do not rename these files** — the runners expect exactly `comments_metadata.json` under `dir_path` and `missing_commentors_from_ldap.json` under `dir_path/users/`. +`prepare` is optional: it searches the collection for duplicated records (same title, description and file checksums, and `981__a` merges) and writes the legacy recids to redirect in `duplicated_pids.json` (and the EP report numbers shared with other records in `ep_report_number_dups.json`) in the working directory of the collection. Run it if you want to test the redirections: when `duplicated_pids.json` is there, the migration reads it, otherwise nothing is redirected. It needs `analyze_collection_migration.py` and the legacy Invenio, like the dumps, and for several queries the `analyze_query` of the collection configuration. -2. Ensure files are in place (paths should match `streams.yaml`): +The dumps change only when the legacy records change (f.e. when curators intervene) , so they do not need to be refreshed on every run. `dump files` is not needed locally. -- `comments_metadata.json` under the collection `dir_path` -- `people.csv` and `missing_commentors_from_ldap.json` under `dir_path/users/` +## 4. Environment -```shell -cp /eos/media/cds/cds-rdm//migration/users/people.csv /eos/media/cds/cds-rdm//migration//comments/users/ -``` +Nothing to set: `CDS_MIGRATOR_KIT_ENV` is `local` and the data root is `CDS_MIGRATOR_KIT_LOCAL_MIGRATION_ROOT` (`migration_cli/rdm-data` of the repository) by default in `migration_config.py`. To use another folder, export `INVENIO_CDS_MIGRATOR_KIT_MIGRATION_ROOT` with an absolute path, or a path relative to the repository root. -3. Pre-create commenter accounts (dry run first): +The root holds `users/` and the collection folders. The configuration is read from `migration_cli/collections/` of the checkout (`INVENIO_CDS_MIGRATOR_KIT_COLLECTIONS_DIR` to change it). Logs and the records state file go to `INVENIO_CDS_MIGRATOR_KIT_LOGS_PATH` (default `cds_migrator_kit/tmp/logs/`). -```shell -# on migration pod -invenio migration comments commenters-run --collection --dry-run -invenio migration comments commenters-run --collection -``` +## 5. Migrate -After running, make sure the users are indexed and not only created in the DB. - -4. Migrate comments (dry run first): +Try each command with `--dry-run` first. ```shell -# on migration pod -invenio migration comments run --collection --dry-run -invenio migration comments run --collection -``` +invenio migration users submitters run --collection antares +invenio migration run --collection antares -The comments runner can be re-run after errors; already migrated comments are skipped. - -### Migrate the statistics for the successfully migrated records - -When the `invenio migration run` command ends it will produce a `rdm_records_state.json` file which has linked information about the migrated records and the old system. The format will be similar to below: - -```json -{ - "legacy_recid": "2884810", - "parent_recid": "zts3q-6ef46", - "parent_object_uuid": "155be22f-3038-49e0-9f17-9518eaac783a", - "latest_version": "1mae4-skq89", - "latest_version_object_uuid": "155be22f-3038-49e0-9f17-9518eaac783a", - "versions": [ - { - "new_recid": "1mae4-skq89", - "version": 2, - "files": [ - { - "legacy_file_id": 1568736, - "bucket_id": "155be22f-3038-49e0-9f17-9518eaac783a", - "file_key": "Summer student program report.pdf", - "file_id": "06cdb9d2-635f-4dbe-89fe-4b27afddeaa2", - "size": "1690854" - } - ] - } - ] -} +# only if you dumped the comments +invenio migration users commenters run --collection antares +invenio migration comments run --collection antares ``` -- Open the `cds_migrator_kit/rdm/migration/stats/config.py` and - - - export the below 2 environmental variables - - `CDS_MIGRATOR_KIT_SRC_SEARCH_HOSTS`: e.g `export CDS_MIGRATOR_KIT_SRC_SEARCH_HOSTS='[{"host": "os-cds-legacy.cern.ch", "url_prefix": "/os", "timeout": 30, "port": 443, "use_ssl": true, "verify_certs": false, "http_auth": ["LEGACY_PRODUCTION_OPENSEARCH_USERNAME", ""]}]'` - - you find the credentials for `LEGACY_PRODUCTION_OPENSEARCH_PASSWORD` by `tbag show LEGACY_PRODUCTION_OPENSEARCH_PASSWORD --hg cds` - - you find the credentials for `LEGACY_PRODUCTION_OPENSEARCH_USERNAME` by `tbag show LEGACY_PRODUCTION_OPENSEARCH_USERNAME --hg cds` - -- Open a shell and run the following commands +Then reindex, to see the missing user accounts in the UI: -```bash -$ invenio migration stats run --filepath "path/to/file/of/rdm_records_state.json" +```shell +invenio rdm rebuild-all-indices -o users ``` -This will migrate only the raw statistic events. When all events are ingested to the new cluster then we will need to aggregate them. - -To do so, you need to run after you have set the correct bookmark for each event: - -on opensearch +The migration report can be browsed with: ```shell -DELETE /cds-rdm-stats-bookmarks - -POST /cds-rdm-stats-bookmarks/_doc -{ - "date": "2000-06-26T15:56:05.755394", - "aggregation_type": "file-download-agg" -} -POST /cds-rdm-stats-bookmarks/_doc -{ - "date": "2000-06-26T15:56:05.755394", - "aggregation_type": "record-view-agg" -} -POST /cds-rdm-stats-bookmarks/_doc -{ - "date": "2000-06-26T15:56:05.755394", - "aggregation_type": "stats_reindex" -} -``` - +gunicorn -b :8080 --timeout 120 --graceful-timeout 60 cds_migrator_kit.app:app --reload --log-level debug --capture-output --access-logfile '-' --error-logfile '-' ``` -from invenio_stats.tasks import aggregate_events -start_date = '2000-01-01' -end_date = '2024-12-01' +## 6. Statistics (optional in local) -aggregations = ["record-view-agg", "file-download-agg"] -aggregate_events(aggregations) +The legacy events are read from the legacy OpenSearch, so this needs access to it: export `CDS_MIGRATOR_KIT_SRC_SEARCH_HOSTS` as described in `migration_cli/step_by_step_documentation.md`, then: -from invenio_rdm_records.services.tasks import reindex_stats -stats_indices = [ "stats-record-view", "stats-file-download",] - -reindex_stats(stats_indices) +```shell +invenio migration stats events run --collection antares +invenio migration stats aggregate ``` -visit https://migration-cds-rdm-dev.app.cern.ch for report - -## Local rerun migration from clean state without setup everything again - -If you want to cleanup a previous migration run without having to re setup everything -i.e not repopulating all vocabularies which takes a lot of time, then run the following -recipe: - -- Cleanup db tables from pgadmin (**!!! ONLY ON DEV or local !!!**) - -```sql -DELETE FROM rdm_versions_state; -DELETE FROM rdm_records_files; -DELETE FROM rdm_drafts_files; -DELETE FROM rdm_records_metadata; -DELETE FROM rdm_drafts_metadata; -DELETE FROM rdm_parents_metadata; -DELETE FROM communities_files; -DELETE FROM oaiserver_set; -DELETE FROM communities_metadata; -DELETE FROM files_objecttags; -DELETE FROM files_object; -DELETE FROM files_buckettags; -DELETE FROM files_bucket; -DELETE FROM files_files; -DELETE FROM pidstore_pid WHERE pid_type = 'lrecid'; -DELETE FROM pidstore_pid WHERE pid_type = 'recid'; -DELETE FROM pidstore_pid WHERE pid_type = 'doi'; -DELETE FROM cds_clc_record_sync; -DELETE FROM cds_migration_legacy_records; -``` +## 7. Rerun from a clean state -- Cleanup indexed documents from opensearch +Restore the clean-slate snapshot taken in step 0, then run step 5 again: -``` -POST /cds-rdm-dev-rdmrecords/_delete_by_query -{ - "query": { - "match_all": {} - } -} -POST /cds-rdm-dev-communities/_delete_by_query -{ - "query": { - "match_all": {} - } -} +```shell +scripts/snapshot.sh restore clean-slate ``` -- Rerun `invenio migration community dump --slug 'sspn' --title 'Summer Student Project Notes' --filepath /path/to/cds_migrator_kit/rdm/migration/streams.yaml` -- Rerun `invenio migration run` +It restores PostgreSQL and OpenSearch to the state before the migration, and resets the file location of the instance. This replaces the manual cleanup described in `migration_cli/step_by_step_documentation.md` ("Local rerun migration from clean state"), which is only needed when there is no snapshot. Only do it on a local instance. If the snapshot is old, update it first as described in step 0. diff --git a/cds_migrator_kit/rdm/cli.py b/cds_migrator_kit/rdm/cli.py index cea59734..ad3b4e77 100644 --- a/cds_migrator_kit/rdm/cli.py +++ b/cds_migrator_kit/rdm/cli.py @@ -8,13 +8,15 @@ """CDS-RDM command line module.""" import logging -from datetime import datetime +import os +from datetime import date, datetime from pathlib import Path import click from flask import current_app from flask.cli import with_appcontext +from cds_migrator_kit.rdm import collection_conf from cds_migrator_kit.rdm.affiliations.runner import RecordAffiliationsRunner from cds_migrator_kit.rdm.affiliations.streams import AffiliationsStreamDefinition from cds_migrator_kit.rdm.comments.runner import CommenterRunner, CommentsRunner @@ -39,6 +41,91 @@ cli_logger = logging.getLogger("migrator") +BOOKMARK_START_DATE = "2000-06-26T15:56:05.755394" + + +def _env_and_root(): + """Environment the CLI runs in (CDS_MIGRATOR_KIT_ENV) and the data root.""" + env = current_app.config["CDS_MIGRATOR_KIT_ENV"] + root = current_app.config.get("CDS_MIGRATOR_KIT_MIGRATION_ROOT") + if env == "local" and not root: + root = current_app.config.get("CDS_MIGRATOR_KIT_LOCAL_MIGRATION_ROOT") + return env, root + + +def _resolve_community(slug): + """Id of the community with the given slug in this instance.""" + from invenio_access.permissions import system_identity + from invenio_communities.proxies import current_communities + from invenio_pidstore.errors import PIDDoesNotExistError + + try: + return str(current_communities.service.read(system_identity, slug)._record.id) + except PIDDoesNotExistError: + raise click.UsageError( + f"The community `{slug}` does not exist in this environment: create it first." + ) + + +def _collection_config(collection): + """Config of the collection from migration_cli/collections/.yaml. + + Returns None when the collection has none (it is then read from the + streams.yaml given by CDS_MIGRATOR_KIT_STREAM_CONFIG). + """ + try: + env, root = _env_and_root() + except click.UsageError: + return None # local setup, no EOS folder: streams.yaml only + try: + return collection_conf.build_stream_config( + collection, + env, + db_uri=current_app.config["SQLALCHEMY_DATABASE_URI"], + resolve_community=_resolve_community, + root=root, + confs_dir=current_app.config.get("CDS_MIGRATOR_KIT_COLLECTIONS_DIR"), + ) + except FileNotFoundError: + return None + + +def _config_for(collection, section="records"): + """Config of the collection, or None to use the entry of streams.yaml. + + Fails with a clear message when the collection is in neither. + """ + config = _collection_config(collection) + if config is not None: + return config + import yaml + + with open(current_app.config["CDS_MIGRATOR_KIT_STREAM_CONFIG"]) as fp: + entries = (yaml.safe_load(fp) or {}).get(section) or {} + if collection not in entries: + confs_dir = ( + current_app.config.get("CDS_MIGRATOR_KIT_COLLECTIONS_DIR") + or collection_conf.default_confs_dir() + ) + raise click.UsageError( + f"No configuration for the collection `{collection}`: add " + f"{os.path.join(confs_dir, collection + '.yaml')}" + ) + return None + + +def _collection_dirs(collection): + """Return the (dump dir, shared users dir) of a collection.""" + env, root = _env_and_root() + try: + conf = collection_conf.read_conf( + collection, current_app.config.get("CDS_MIGRATOR_KIT_COLLECTIONS_DIR") + ) + except FileNotFoundError as exc: + raise click.UsageError(str(exc)) + base = collection_conf.working_dir(conf, env, root) + return os.path.join(base, "dump"), collection_conf.users_dir(env, root) + @click.group() def migration(): @@ -82,6 +169,7 @@ def run(collection, dry_run=False, keep_logs=False, workers=None): collection=collection, keep_logs=keep_logs, workers=workers, + config=_config_for(collection, "records"), ) runner.run() @@ -93,23 +181,8 @@ def stats(): pass -@stats.command() -@click.option( - "--dry-run", - is_flag=True, -) -@click.option( - "--filepath", - help="Path to the list of records file that the legacy statistics will be migrated.", -) -@click.option( - "--less-than-date", - default=lambda: datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), - help="ISO string e.g 2024-12-13T20:00:00 to migrate events up to this date.", -) -@with_appcontext -def run(filepath, less_than_date, dry_run=False): - """Migrate the legacy statistics for the records in `filepath`.""" +def _run_stats_events(filepath, less_than_date, dry_run): + """Migrate the raw legacy statistics events of the records in `filepath`.""" stream_config = current_app.config["CDS_MIGRATOR_KIT_RECORD_STATS_STREAM_CONFIG"] stream_config["DEST_SEARCH_INDEX_PREFIX"] = ( f"{current_app.config['SEARCH_INDEX_PREFIX']}events-stats" @@ -127,6 +200,110 @@ def run(filepath, less_than_date, dry_run=False): runner.run() +_less_than_date_option = click.option( + "--less-than-date", + default=lambda: datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + help="ISO string e.g 2024-12-13T20:00:00 to migrate events up to this date.", +) + + +@stats.command() +@click.option( + "--dry-run", + is_flag=True, +) +@click.option( + "--filepath", + help="Path to the list of records file that the legacy statistics will be migrated.", +) +@_less_than_date_option +@with_appcontext +def run(filepath, less_than_date, dry_run=False): + """Migrate the legacy statistics for the records in `filepath`.""" + _run_stats_events(filepath, less_than_date, dry_run) + + +@stats.group() +def events(): + """Raw statistics events.""" + pass + + +@events.command("run") +@click.option( + "--dry-run", + is_flag=True, +) +@click.option( + "--collection", + help="Collection whose records' legacy events are migrated.", + required=True, +) +@click.option( + "--filepath", + help="Records state file. Default: the one written by `migration run` " + "for the collection.", +) +@_less_than_date_option +@with_appcontext +def events_run(collection, filepath, less_than_date, dry_run=False): + """Migrate the legacy statistics events of a migrated collection.""" + if not filepath: + filepath = os.path.join( + current_app.config["CDS_MIGRATOR_KIT_LOGS_PATH"], + collection, + "rdm_records_state.json", + ) + if not os.path.isfile(filepath): + raise click.UsageError(f"Records state file not found: {filepath}") + _run_stats_events(filepath, less_than_date, dry_run) + + +@stats.command() +@click.option( + "--start-date", + default=BOOKMARK_START_DATE, + show_default=True, + help="Date the aggregation bookmarks are reset to.", +) +@click.option( + "--end-date", + default=lambda: date.today().isoformat(), + help="Aggregate events up to this date. Default: today.", +) +@with_appcontext +def aggregate(start_date, end_date): + """Reset the aggregation bookmarks, aggregate the events and reindex stats. + + Run it once all the events of all the collections are ingested. + """ + from invenio_rdm_records.services.tasks import reindex_stats + from invenio_search import current_search_client + from invenio_search.utils import prefix_index + from invenio_stats.tasks import aggregate_events + + bookmarks_index = prefix_index("stats-bookmarks") + if not current_search_client.indices.exists(index=bookmarks_index): + raise click.UsageError(f"Index {bookmarks_index} does not exist.") + + click.echo(f"Resetting the bookmarks in {bookmarks_index} to {start_date}") + current_search_client.delete_by_query( + index=bookmarks_index, body={"query": {"match_all": {}}}, refresh=True + ) + for aggregation_type in ("file-download-agg", "record-view-agg", "stats_reindex"): + current_search_client.index( + index=bookmarks_index, + body={"date": start_date, "aggregation_type": aggregation_type}, + refresh=True, + ) + + click.echo(f"Aggregating events up to {end_date}") + aggregate_events(["record-view-agg", "file-download-agg"], end_date=end_date) + + click.echo("Reindexing the stats") + reindex_stats(["stats-record-view", "stats-file-download"]) + + @migration.group() def users(): """Migration CLI for users by collection.""" @@ -188,6 +365,49 @@ def submitters_run(dirpath, users_dir_path, dry_run=False): runner.run() +@users.group() +def submitters(): + """Submitters accounts of a collection.""" + pass + + +@submitters.command("run") +@click.option("--dry-run", is_flag=True) +@click.option( + "--collection", + help="Collection whose record submitters are created.", + required=True, +) +@with_appcontext +def submitters_collection_run(collection, dry_run=False): + """Create the missing submitters accounts of a collection.""" + dirpath, users_dir_path = _collection_dirs(collection) + ctx = click.get_current_context() + ctx.invoke( + submitters_run, dirpath=dirpath, users_dir_path=users_dir_path, dry_run=dry_run + ) + + +@users.group() +def commenters(): + """Commenters accounts of a collection.""" + pass + + +@commenters.command("run") +@click.option("--dry-run", is_flag=True) +@click.option( + "--collection", + help="Collection whose commenters are created.", + required=True, +) +@with_appcontext +def commenters_collection_run(collection, dry_run=False): + """Create the commenters accounts of a collection.""" + ctx = click.get_current_context() + ctx.invoke(commenters_run, collection=collection, dry_run=dry_run) + + @migration.group() def affiliations(): """Migration CLI for affiliations.""" @@ -289,6 +509,7 @@ def run(collection, dry_run=False): log_dir=log_dir, collection=collection, dry_run=dry_run, + config=_config_for(collection, "comments"), ) runner.run() @@ -314,5 +535,6 @@ def commenters_run(collection, dry_run=False): log_dir=log_dir, collection=collection, dry_run=dry_run, + config=_config_for(collection, "comments"), ) runner.run() diff --git a/cds_migrator_kit/rdm/collection_conf.py b/cds_migrator_kit/rdm/collection_conf.py new file mode 100644 index 00000000..e6e4ac75 --- /dev/null +++ b/cds_migrator_kit/rdm/collection_conf.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-Migrator-Kit is free software; you can redistribute it and/or modify +# it under the terms of the MIT License; see LICENSE file for more details. + +"""Per-collection migration configuration. + +The configuration of a collection is a ``.yaml`` file kept in git, +in ``migration_cli/collections/``. It does not depend on the environment, so +the same file is used on every environment (and by the legacy script on +``cds-migration-01``). It replaces the per-collection entries of +``streams.yaml``. + +The data of a collection lives apart, under the root of the migration data of +the environment: + + / /eos/media/cds/cds-rdm//migration + users/ people.csv, shared by all collections + / the working directory + dump/ files/ duplicated_pids.json + comments/ comments_metadata.json, users/, / + +The root is, in this order: given explicitly (``--root`` or the +``CDS_MIGRATOR_KIT_MIGRATION_ROOT`` config), else the EOS path of the +environment. ``local`` has no default: the CLI gives it +``CDS_MIGRATOR_KIT_LOCAL_MIGRATION_ROOT`` (``migration_cli/rdm-data``, relative +to the root of the repository). + +``migration_cli/collection_migration.py`` (legacy host) writes the data and +``migration_cli/migrating-a-collection.md`` describes the whole flow. +""" + +import os + +import yaml + +EOS_ROOT = "/eos/media/cds/cds-rdm/{env}/migration" +DUPLICATES_FILENAME = "duplicated_pids.json" + +# names an environment may be deployed under -> EOS folder +ENV_FOLDERS = {"production": "prod"} + +# flags that map 1:1 to a key of the collection entry +_RECORD_FLAGS = ( + "restricted", + "plots", + "create_inclusion_request", + "update_new_version_publication_date", + "access_grants_view", + "preferred_model", +) + + +def env_folder(env): + """EOS folder name of an environment.""" + return ENV_FOLDERS.get(env, env) + + +def default_confs_dir(): + """The ``migration_cli/collections`` folder of the checkout.""" + here = os.path.dirname(os.path.abspath(__file__)) + return os.path.normpath( + os.path.join(here, "..", "..", "migration_cli", "collections") + ) + + +def repo_root(): + """Root of the repository (checkout) the package runs from.""" + here = os.path.dirname(os.path.abspath(__file__)) + return os.path.normpath(os.path.join(here, "..", "..")) + + +def migration_root(env, root=None): + """Root folder of the migration data for the environment. + + A relative ``root`` is relative to the root of the repository, so it does + not depend on the directory the command is run from. + """ + if root: + return os.path.normpath(os.path.join(repo_root(), root)) + if env == "local": + raise ValueError( + "`local` has no default root of the migration data: set " + "CDS_MIGRATOR_KIT_MIGRATION_ROOT or CDS_MIGRATOR_KIT_LOCAL_MIGRATION_ROOT" + ) + return EOS_ROOT.format(env=env_folder(env)) + + +def users_dir(env, root=None): + """Folder with the people.csv shared by all collections.""" + return os.path.join(migration_root(env, root), "users") + + +def conf_path(collection, confs_dir=None): + """Path of the configuration file of the collection.""" + return os.path.join(confs_dir or default_confs_dir(), collection + ".yaml") + + +def read_conf(collection, confs_dir=None): + """Read and validate the configuration of the collection.""" + path = conf_path(collection, confs_dir) + if not os.path.isfile(path): + raise FileNotFoundError( + "No configuration for `{}`: {}".format(collection, path) + ) + with open(path) as fp: + conf = yaml.safe_load(fp) or {} + if conf.get("name") != collection: + raise ValueError( + "{} is the configuration of `{}`, not of `{}`".format( + path, conf.get("name"), collection + ) + ) + conf.setdefault("flags", {}) + return conf + + +def working_dir(conf, env, root=None): + """Working directory of the collection (``folder`` groups collections).""" + return os.path.join(migration_root(env, root), conf.get("folder", conf["name"])) + + +def build_stream_config( + collection, env, db_uri, resolve_community, root=None, confs_dir=None +): + """Build the equivalent of a streams.yaml for one collection. + + ``resolve_community`` turns the slug of a community into its id in the + current instance. The result has the layout the runners expect (``records`` + and ``comments`` sections). + """ + conf = read_conf(collection, confs_dir) + base = working_dir(conf, env, root) + flags = conf["flags"] + people_dir = users_dir(env, root) + + slugs = conf.get("communities") + if not slugs: + raise ValueError( + "`communities` (slugs) is not defined for `{}`".format(collection) + ) + + record = {"data_dir": base} + for flag in _RECORD_FLAGS: + if flag in flags: + record[flag] = flags[flag] + record["extract"] = {"dirpath": os.path.join(base, "dump")} + record["transform"] = { + "files_dump_dir": os.path.join(base, "files") + "/", + "missing_users": people_dir, + "communities_ids": [resolve_community(slug) for slug in slugs], + } + if flags.get("workers"): + record["transform"]["workers"] = flags["workers"] + + duplicates = os.path.join(base, DUPLICATES_FILENAME) + if os.path.isfile(duplicates): + record["load"] = {"legacy_pids_to_redirect": duplicates} + + comments = { + "dir_path": os.path.join(base, "comments"), + "users_dir": people_dir, + "reviewers": conf.get("reviewers", []), + } + return { + "db_uri": db_uri, + "records": {collection: record}, + "comments": {collection: comments}, + } diff --git a/cds_migrator_kit/rdm/comments/runner.py b/cds_migrator_kit/rdm/comments/runner.py index 17319927..51b86ecf 100644 --- a/cds_migrator_kit/rdm/comments/runner.py +++ b/cds_migrator_kit/rdm/comments/runner.py @@ -27,10 +27,16 @@ class CommentsRunner: """ETL streams runner.""" def __init__( - self, stream_definition, config_filepath, log_dir, collection, dry_run + self, + stream_definition, + config_filepath, + log_dir, + collection, + dry_run, + config=None, ): """Constructor.""" - config = read_config(config_filepath) + config = config or read_config(config_filepath) collection_config = config["comments"][collection] self.log_dir = Path(log_dir) @@ -68,13 +74,22 @@ class CommenterRunner: """ETL streams runner dedicated to pre-create commenters accounts.""" def __init__( - self, stream_definition, config_filepath, log_dir, collection, dry_run + self, + stream_definition, + config_filepath, + log_dir, + collection, + dry_run, + config=None, ): """Constructor.""" - config = read_config(config_filepath) + config = config or read_config(config_filepath) collection_config = config["comments"][collection] dirpath = collection_config["dir_path"] - missing_users_dir = os.path.join(dirpath, "users") + # missing_commentors_from_ldap.json is always under /users, + # people.csv can be read from the shared users folder + ldap_users_dir = os.path.join(dirpath, "users") + missing_users_dir = collection_config.get("users_dir", ldap_users_dir) filename = "missing_commentors_from_ldap.json" self.log_dir = Path(log_dir) @@ -85,12 +100,13 @@ def __init__( self.stream = Stream( stream_definition.name, extract=stream_definition.extract_cls( - os.path.join(missing_users_dir, filename) + os.path.join(ldap_users_dir, filename) ), transform=stream_definition.transform_cls(), load=stream_definition.load_cls( dry_run=dry_run, missing_users_dir=missing_users_dir, + missing_ldap_users_dir=ldap_users_dir, missing_ldap_users_filename=filename, logger=self.logger, user_api_cls=CDSMigrationUserAPI, diff --git a/cds_migrator_kit/rdm/data/cms-notes/.gitkeep b/cds_migrator_kit/rdm/data/cms-notes/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/cds_migrator_kit/rdm/data/summer_student_reports/.gitkeep b/cds_migrator_kit/rdm/data/summer_student_reports/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/cds_migrator_kit/rdm/data/summer_student_reports/duplicated_pids.json b/cds_migrator_kit/rdm/data/summer_student_reports/duplicated_pids.json deleted file mode 100644 index 8d02028c..00000000 --- a/cds_migrator_kit/rdm/data/summer_student_reports/duplicated_pids.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "2913580": 2913581, - "2907803": 2907804, - "2909209": 2909210, - "2277344": 2277345, - "2210817": 2210692, - "1594978": 1594977, - "2907269": 2907268, - "2651079": 2649734, - "2834391": 2834390, - "2827084": 2827466, - "1752239": 1752513, - "1594994": 1594988, - "1570389": 1570388, - "2913366": 2914602, - "1753847": 1753846, - "2778360": 2778413, - "1595035": 1595023, - "2634869": 2634868 -} diff --git a/cds_migrator_kit/rdm/migration_config.py b/cds_migrator_kit/rdm/migration_config.py index 29ca7c73..73c332a8 100644 --- a/cds_migrator_kit/rdm/migration_config.py +++ b/cds_migrator_kit/rdm/migration_config.py @@ -352,6 +352,23 @@ def _(x): # needed to avoid start time failure with lazy strings CDS_MIGRATOR_KIT_STREAM_CONFIG = "cds_migrator_kit/rdm/streams.yaml" +CDS_MIGRATOR_KIT_COLLECTIONS_DIR = None +"""Folder with the per-collection configuration files. Defaults to +migration_cli/collections of the checkout; set it when the package is not +installed from the checkout (e.g. INVENIO_CDS_MIGRATOR_KIT_COLLECTIONS_DIR).""" + +CDS_MIGRATOR_KIT_MIGRATION_ROOT = None +"""Root of the migration data (users/ and one folder per collection). Leave it +unset on dev, sandbox and prod: it defaults to the EOS path of +CDS_MIGRATOR_KIT_ENV, and a value here would override it. A relative path is +relative to the root of the repository.""" + +CDS_MIGRATOR_KIT_LOCAL_MIGRATION_ROOT = "migration_cli/rdm-data" +"""Root of the migration data when CDS_MIGRATOR_KIT_ENV is `local` and +CDS_MIGRATOR_KIT_MIGRATION_ROOT is not set. Other environments ignore it. +A relative path is relative to the root of the repository, not to the current +directory.""" + RDM_RECORDS_IDENTIFIERS_SCHEMES = { "cdsrn": { "label": _("CDS Reference"), diff --git a/cds_migrator_kit/rdm/readme_models.md b/cds_migrator_kit/rdm/readme_models.md new file mode 100644 index 00000000..c0f57b8f --- /dev/null +++ b/cds_migrator_kit/rdm/readme_models.md @@ -0,0 +1,63 @@ +# Migration model manual + +## Dump users + +! Attention If you need to dump the users from legacy DB or you need to process the people collection + +https://gitlab.cern.ch/cds-team/production_scripts/-/blob/master/cds-rdm/migration/dump_users.py?ref_type=heads + + +## Dump a subset of records on legacy + +on webnode: `cds-migration-01` + +```bash +inveniomigrator dump records -q '980:INTNOTECMSPUBL 980:NOTE -980:DELETED' --file-prefix cms-notes --chunk-size=1000 + +``` + +## Define transforming models and rules resolution + +You can adapt XML processing to different subsets of records by implementing different data models for each subset (f.e. collection). +Let's take CMS notes as an example: + +```python + +class CMSNote(CdsOverdo): + """Translation Index for CDS Books.""" + + __query__ = ( + '980__:INTNOTECMSPUBL 980__:NOTE' + ) + + __model_ignore_keys__ = {} + + _default_fields = {} + + + +model = CMSNote(bases=(), + entry_point_group="cds_rdm.migrator.rules" + ) + +``` + +**query** - defines the MARC fields to which specific record should match. Attention: It does not recognise regexes that are used to specify the collection query in the admin interface of legacy CDS. + +\***\*model_ignore_keys\*\*** - set of keys to be ignored for this data model - fields will not be migrated + +**bases** - by defining bases of models you can specify a parent model which fits all the subsets of records (f.e. 245 - title field MARC to JSON translation could be the same for all the models) + +**entry_point_group** - reference to where the model should lookup for the set of the MARC translation rules, see the entrypoints below. + +After defining your model and set of rules,you have to register them in the entrypoints of your application, in setup.cfg: + +```editorconfig + +[options.entry_points] +cds_migrator_kit.migrator.models = + cms_note = cds_rdm.migration.transform.models.note:model +cds_migrator_kit.migrator.rules = + base = cds_rdm.migration.transform.xml_processing.rules + +``` diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index 28d64b40..5b27a06d 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -180,143 +180,6 @@ records: communities_ids: - "2ca82423-148a-42b5-86ff-03336b961c8e" - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - antares: - data_dir: cds_migrator_kit/rdm/data/former_exp/antares - plots: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/antares - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "e2d2ed70-8a3c-453b-aa5b-1fa985f0906b" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - antares_restr: - data_dir: cds_migrator_kit/rdm/data/former_exp/antares_restr - restricted: "True" - plots: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/antares_restr - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "e2d2ed70-8a3c-453b-aa5b-1fa985f0906b" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - dirac: - plots: true - data_dir: cds_migrator_kit/rdm/data/former_exp/dirac - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/dirac - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "6a289642-5378-4daf-87b5-bb58af00487a" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - harp_cdp: - plots: true - data_dir: cds_migrator_kit/rdm/data/former_exp/harp_cdp - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/harp_cdp - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "2541c76f-6c9b-415b-9a47-e814ec01f576" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - imxgam: - plots: true - data_dir: cds_migrator_kit/rdm/data/former_exp/imXgam - restricted: "True" - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/imXgam - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "88a5cdf4-974a-44d4-b145-d19ee6346bb8" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - re29: - data_dir: cds_migrator_kit/rdm/data/former_exp/re29 - plots: true - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/re29 - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "aff67e22-9027-4401-a16f-1aaac2cd2c27" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - re29_restr: - data_dir: cds_migrator_kit/rdm/data/former_exp/re29_restr - plots: true - restricted: "True" - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/re29_restr - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "aff67e22-9027-4401-a16f-1aaac2cd2c27" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - ua2: - plots: true - data_dir: cds_migrator_kit/rdm/data/former_exp/ua2 - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/ua2 - preferred_model: research - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "4040d2c7-7648-4f1a-81fa-7112fec8d130" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - ua4: - data_dir: cds_migrator_kit/rdm/data/former_exp/ua4 - plots: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/ua4 - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "1b9239a7-5938-44e3-8ea8-2c530627f659" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - ua5: - data_dir: cds_migrator_kit/rdm/data/former_exp/ua5 - plots: true - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/ua5 - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "7996b881-2420-4791-ac17-00cf3c1e7196" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - ua8: - plots: true - data_dir: cds_migrator_kit/rdm/data/former_exp/ua8 - extract: - dirpath: cds_migrator_kit/rdm/data/former_exp/ua8 - transform: - files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "2dbeb56c-b4ab-4cad-8d82-5a1f90499bd7" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - fap: - data_dir: cds_migrator_kit/rdm/data/fap - restricted: "True" - extract: - dirpath: cds_migrator_kit/rdm/data/fap/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/fap/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "" at: data_dir: cds_migrator_kit/rdm/data/at restricted: "False" @@ -339,7 +202,7 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/isolde/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: - - "77d6a84a-596e-414f-817b-1c43e9c66258" + - "1c46ceab-af68-4502-a436-cdf5a4cfbbe0" load: legacy_pids_to_redirect: cds_migrator_kit/rdm/data/isolde/duplicated_pids.json hie_isolde: @@ -394,34 +257,6 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "d9c138bd-d39c-4797-a06c-c6f8db45f3ce" - aleph_missing: - data_dir: cds_migrator_kit/rdm/data/lep_missing/aleph - plots: true - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/lep_missing/aleph/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/lep_missing/aleph/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "7c568753-550b-4b48-8d76-461181973100" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/lep_missing/aleph/duplicated_pids.json - l3_missing: - data_dir: cds_migrator_kit/rdm/data/lep_missing/l3 - plots: true - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/lep_missing/l3/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/lep_missing/l3/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "bb0ac2d2-b90b-498e-bf64-1986791d1032" - - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/lep_missing/l3/duplicated_pids.json ab_dep: data_dir: cds_migrator_kit/rdm/data/dep_ab plots: true @@ -590,39 +425,6 @@ records: - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" load: legacy_pids_to_redirect: cds_migrator_kit/rdm/data/div_msc/duplicated_pids.json - lhcf: - data_dir: cds_migrator_kit/rdm/data/lhcf - plots: true - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/lhcf/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/lhcf/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "31bf228a-ae6e-462b-b70e-345d3e74b8ca" - lhcf-ep: - data_dir: cds_migrator_kit/rdm/data/lhcf-ep - plots: true - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/lhcf-ep/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/lhcf-ep/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "88a105fe-4713-493b-b555-6ab398599d21" - na58: - data_dir: cds_migrator_kit/rdm/data/north_area/na58 - plots: true - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/north_area/na58/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/north_area/na58/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "742fea6c-0685-4da2-8000-c1f19635ba86" na61: data_dir: cds_migrator_kit/rdm/data/north_area/na61 plots: true diff --git a/cds_migrator_kit/rdm/streams_done.yaml b/cds_migrator_kit/rdm/streams_done.yaml index 68b97636..ceac9fc1 100644 --- a/cds_migrator_kit/rdm/streams_done.yaml +++ b/cds_migrator_kit/rdm/streams_done.yaml @@ -298,63 +298,3 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "e849139c-dff5-45a7-90b1-90cdddc9c00e" - faser-drafts: - data_dir: cds_migrator_kit/rdm/data/faser-drafts - restricted: "True" - create_inclusion_request: true - access_grants_view: - - faser-all - extract: - dirpath: cds_migrator_kit/rdm/data/faser-drafts/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/faser-drafts/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "33af9368-5bad-45cb-9360-8c9e5dfca09f" - faser: - data_dir: cds_migrator_kit/rdm/data/faser - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/faser/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/faser/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "33af9368-5bad-45cb-9360-8c9e5dfca09f" - faser-ep: - data_dir: cds_migrator_kit/rdm/data/faser - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/faser-ep/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/faser/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "e5de962b-5e2c-408f-be69-2823fda1f732" - ship: - data_dir: cds_migrator_kit/rdm/data/ship - plots: true - create_inclusion_request: true - extract: - dirpath: cds_migrator_kit/rdm/data/ship/dump - transform: - files_dump_dir: cds_migrator_kit/rdm/data/ship/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "8b213ed1-31a6-408a-b474-c0c39734c2e3" - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/ship/legacy_pids_to_redirect.json - ship-restricted: - data_dir: cds_migrator_kit/rdm/data/ship-restricted - plots: true - create_inclusion_request: true - restricted: "True" - extract: - dirpath: cds_migrator_kit/rdm/data/ship-restricted/dump/ - transform: - files_dump_dir: cds_migrator_kit/rdm/data/ship-restricted/files/ - missing_users: cds_migrator_kit/rdm/data/users - communities_ids: - - "8b213ed1-31a6-408a-b474-c0c39734c2e3" - load: - legacy_pids_to_redirect: cds_migrator_kit/rdm/data/ship-restricted/legacy_pids_to_redirect.json diff --git a/cds_migrator_kit/rdm/streams_shelved.yaml b/cds_migrator_kit/rdm/streams_shelved.yaml index 371e50a6..f5477c31 100644 --- a/cds_migrator_kit/rdm/streams_shelved.yaml +++ b/cds_migrator_kit/rdm/streams_shelved.yaml @@ -11,6 +11,7 @@ records: files_dump_dir: cds_migrator_kit/rdm/data/former_exp/files/ missing_users: cds_migrator_kit/rdm/data/users communities_ids: + - "c2c46ab3-5fb4-4d86-83c6-5d9dc8392d6f" - "adc02716-780b-4bba-8e89-6316c9a11cf0" lcd: plots: true diff --git a/cds_migrator_kit/reports/log.py b/cds_migrator_kit/reports/log.py index c8cf7a27..142d4e00 100644 --- a/cds_migrator_kit/reports/log.py +++ b/cds_migrator_kit/reports/log.py @@ -160,8 +160,12 @@ def add_information(self, recid, state): """ if recid in self._temp_state_cache: new_state = deepcopy(self._temp_state_cache[recid]) - new_state["message"] = f"{new_state.get('message', '')}\n{state.get('message', '')}" - new_state["value"] = f"{new_state.get('value', '')}\n{state.get('value', '')}" + new_state["message"] = ( + f"{new_state.get('message', '')}\n{state.get('message', '')}" + ) + new_state["value"] = ( + f"{new_state.get('value', '')}\n{state.get('value', '')}" + ) state = new_state self._temp_state_cache[recid] = state diff --git a/cds_migrator_kit/reports/utils.py b/cds_migrator_kit/reports/utils.py index 2b1c49d6..3e74b555 100644 --- a/cds_migrator_kit/reports/utils.py +++ b/cds_migrator_kit/reports/utils.py @@ -7,6 +7,7 @@ # under the terms of the MIT License; see LICENSE file for more details. """CDS Migrator Records utils.""" + import re from fuzzywuzzy import fuzz diff --git a/cds_migrator_kit/runner/runner.py b/cds_migrator_kit/runner/runner.py index 975bf0ce..6c80245d 100644 --- a/cds_migrator_kit/runner/runner.py +++ b/cds_migrator_kit/runner/runner.py @@ -40,9 +40,14 @@ def __init__( collection, keep_logs, workers=None, + config=None, ): - """Constructor.""" - config = self._read_config(config_filepath) + """Constructor. + + ``config`` (already loaded, e.g. built from a migration_conf.yaml) + takes precedence over ``config_filepath``. + """ + config = config or self._read_config(config_filepath) self.collection = collection self.keep_logs = keep_logs self.db_uri = config.get("db_uri") @@ -69,7 +74,9 @@ def __init__( "access_grants_view", False ) self.plots = stream_config[collection].get("plots", False) - self.preferred_model = stream_config[collection].get("preferred_model", None) + self.preferred_model = stream_config[collection].get( + "preferred_model", None + ) self.data_dir.mkdir(parents=True, exist_ok=True) StandardLogger.initialize() @@ -135,8 +142,6 @@ def run(self): raise e finally: elapsed = time.perf_counter() - t0 - perf_logger.info( - f"Stream '{stream.name}' finished in {elapsed:.1f}s" - ) + perf_logger.info(f"Stream '{stream.name}' finished in {elapsed:.1f}s") self.migration_logger.finalise() self.record_state_logger.finalise() diff --git a/cds_migrator_kit/transform/dumper.py b/cds_migrator_kit/transform/dumper.py index a83e87f0..2b072e7d 100644 --- a/cds_migrator_kit/transform/dumper.py +++ b/cds_migrator_kit/transform/dumper.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM MARC XML dumper module.""" + import logging import arrow diff --git a/cds_migrator_kit/transform/xml_processing/quality/parsers.py b/cds_migrator_kit/transform/xml_processing/quality/parsers.py index 67a89bac..07500213 100644 --- a/cds_migrator_kit/transform/xml_processing/quality/parsers.py +++ b/cds_migrator_kit/transform/xml_processing/quality/parsers.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM migration data cleaning module.""" + import re from abc import ABC, abstractmethod diff --git a/cds_migrator_kit/users/load.py b/cds_migrator_kit/users/load.py index 2dc5aa05..4e11e949 100644 --- a/cds_migrator_kit/users/load.py +++ b/cds_migrator_kit/users/load.py @@ -32,12 +32,15 @@ def __init__( logger=logging.getLogger("users"), user_api_cls=None, missing_ldap_users_filename="missing_users.json", + missing_ldap_users_dir=None, ): """Constructor.""" self.dry_run = dry_run self.missing_users_dir = missing_users_dir self.missing_users_filename = missing_users_filename self.missing_ldap_users_filename = missing_ldap_users_filename + # the missing-LDAP-users file can live apart from people.csv + self.missing_ldap_users_dir = missing_ldap_users_dir or missing_users_dir self.dry_run = dry_run self.logger = logger self.user_api_cls = user_api_cls @@ -226,7 +229,7 @@ def get_person(email): def get_person_old_db(email): missing_users_dump = os.path.join( - self.missing_users_dir, self.missing_ldap_users_filename + self.missing_ldap_users_dir, self.missing_ldap_users_filename ) with open(missing_users_dump) as json_file: missing = json.load(json_file) diff --git a/cds_migrator_kit/users/transform.py b/cds_migrator_kit/users/transform.py index 4e501fb0..b22bab82 100644 --- a/cds_migrator_kit/users/transform.py +++ b/cds_migrator_kit/users/transform.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform step module.""" + import logging from copy import deepcopy diff --git a/migration_cli/collection_migration.py b/migration_cli/collection_migration.py new file mode 100644 index 00000000..9ed8d1dd --- /dev/null +++ b/migration_cli/collection_migration.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python +"""Prepare the data of a collection migration (run on `cds-migration-01`). + +Everything is driven by the configuration of the collection, the file +`collections/.yaml` next to this script (see +`collections/_example.yaml`). It does not depend on the environment, and is the +same file the migrator pod reads (`invenio migration ... --collection +`), so there is no streams.yaml entry to add. + +The data goes to the working directory of the collection under the root of the +migration data of the environment: + + /eos/media/cds/cds-rdm//migration// + +The environment is read from the current directory when it is on EOS, or given +with --env. Other roots (e.g. local folders) are given with --root. +The scripts are those of the cds-migrator-kit checkout on the legacy host +(/opt/cds-migrator-kit/migration_cli): this one calls the helpers in subscripts/ +(analyze_collection_migration.py, copy_collection_files.py, +export_collection_comments.py). +See docs in migration_cli/migrating-a-collection.md. + + kinit cdsrdmeosdev # an identity that can write to the EOS folder + cd /eos/media/cds/cds-rdm/dev/migration # or pass --env dev + S=/opt/cds-migrator-kit/migration_cli + + python $S/collection_migration.py dump records --collection antares_restr + python $S/collection_migration.py dump comments --collection antares_restr + python $S/collection_migration.py dump files --collection antares_restr + python $S/collection_migration.py dump users # once per environment + python $S/collection_migration.py prepare --collection antares_restr + +`dump comments` needs the legacy Invenio libraries: run it with the interpreter +that has them (`ipython $S/collection_migration.py -- dump comments ...`). +Python 2 compatible on purpose (legacy host). + +Layout of the working directory: + + dump/ records dump (`dump records`) + comments/ comments_metadata.json, / attachments, + users/missing_commentors_from_ldap.json + files/ legacy files, same relative paths as legacy + files.log one line per copied file + duplicated_pids.json legacy recids to redirect (`prepare`) + ep_report_number_dups.json EP report numbers to review (`prepare`) + +Every step can be re-run: copies are verified (size + checksum) and files that +are already correct are skipped. +""" + +from __future__ import print_function + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys + +PY2 = sys.version_info[0] == 2 + +SUBSCRIPTS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "subscripts") +CONFS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "collections") +EOS_ROOT = "/eos/media/cds/cds-rdm/{env}/migration" +ENV_FOLDERS = {"production": "prod"} +EOS_ROOT_RE = re.compile(r"^(/eos/media/cds/cds-rdm/([^/]+)/migration)(/|$)") +ANALYZE_SCRIPT = os.path.join(SUBSCRIPTS_DIR, "analyze_collection_migration.py") +DUPLICATES_FILENAME = "duplicated_pids.json" +COPY_RETRIES = 3 +LEGACY_FILES_ROOT = "/opt/cdsweb/var/data/files/" + +# Applied to every collection dump. +DUMP_EXCLUSIONS = ( + "-690C_:SCICOM -595:*ARCHIVE* -980:ANNOUNCEMENT -980:DELETED -980:HIDDEN " + "-980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:GREYBOOK" +) +# -980__c:MIGRATED is not excluded on purpose: some records live in several +# communities. + + +# -------------------------------------------------------------------------- +# helpers +# -------------------------------------------------------------------------- +def _fs(path): + """Return `path` in the form the filesystem calls need (py2: utf-8 bytes).""" + if PY2 and isinstance(path, type("")): + return path.encode("utf-8") + return path + + +def _hash(path, algo="sha256"): + digest = hashlib.new(algo) + with open(_fs(path), "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _makedirs(path): + if not os.path.isdir(_fs(path)): + os.makedirs(_fs(path)) + + +def copy_verified(source, destination): + """Copy a file and check size + sha256 on the destination. + + EOS copies can end up truncated, so every copy is read back and compared; + it is retried a few times. Returns False if the destination was already ok. + """ + expected = _hash(source) + if os.path.exists(_fs(destination)) and _hash(destination) == expected: + return False + for attempt in range(1, COPY_RETRIES + 1): + shutil.copyfile(_fs(source), _fs(destination)) + if ( + os.path.getsize(_fs(source)) == os.path.getsize(_fs(destination)) + and _hash(destination) == expected + ): + return True + print( + "Copy verification failed ({}/{}): {}".format( + attempt, COPY_RETRIES, destination + ) + ) + raise IOError("Could not copy {} correctly".format(destination)) + + +class Context(object): + """Working directory, environment and configuration of the collection.""" + + def __init__(self, args, need_conf=True): + cwd = os.getcwd() + match = EOS_ROOT_RE.match(cwd) or EOS_ROOT_RE.match(os.path.realpath(cwd)) + self.env = args.env or (match and match.group(2)) + if not self.env: + sys.exit( + "Cannot tell the environment from the current directory ({}): " + "run from /eos/media/cds/cds-rdm//migration or pass " + "--env".format(cwd) + ) + # root of the migration data: given, else the EOS path of the environment + self.root = args.root or os.environ.get("CDS_MIGRATION_ROOT") + if not self.root: + if self.env == "local": + sys.exit("--root (or CDS_MIGRATION_ROOT) is needed for --env local") + self.root = EOS_ROOT.format(env=ENV_FOLDERS.get(self.env, self.env)) + self.root = self.root.rstrip("/") + self.users = os.path.join(self.root, "users") + self.conf = self._load_conf(args) if need_conf else None + + self.base = ( + os.path.join(self.root, self.conf.get("folder", self.conf["name"])) + if need_conf + else None + ) + if need_conf: + self.dump = os.path.join(self.base, "dump") + self.files = os.path.join(self.base, "files") + self.comments = os.path.join(self.base, "comments") + self.comments_users = os.path.join(self.comments, "users") + self.staging = "/tmp/{}".format(self.conf["name"]) + + def _load_conf(self, args): + if not args.collection: + sys.exit("--collection is required") + path = os.path.join(args.confs_dir, args.collection + ".yaml") + if not os.path.isfile(path): + sys.exit( + "No configuration for `{}`: {} not found".format(args.collection, path) + ) + try: + import yaml + except ImportError: + sys.exit("PyYAML is not installed on this machine") + with open(path) as fp: + conf = yaml.safe_load(fp) or {} + if conf.get("name") != args.collection: + sys.exit( + "{} is the configuration of `{}`, not of `{}`".format( + path, conf.get("name"), args.collection + ) + ) + if not conf.get("queries"): + sys.exit( + "`queries` is not set in {}: fill in the legacy queries".format(path) + ) + conf.setdefault("filename_prefix", conf["name"]) + conf.setdefault("chunk_size", 1000) + conf.setdefault("flags", {}) + return conf + + def ensure_dirs(self): + for path in (self.dump, self.files, self.comments_users): + _makedirs(path) + + def comments_queries(self): + return self.conf.get("comments_queries") or self.conf["queries"] + + +# -------------------------------------------------------------------------- +# dump records +# -------------------------------------------------------------------------- +def _prefixes(conf): + """One file prefix per query, so several queries never overwrite each other.""" + base, count = conf["filename_prefix"], len(conf["queries"]) + if count == 1: + return [base] + return ["{}-{}".format(base, i) for i in range(count)] + + +def cmd_dump_records(ctx, args): + ctx.ensure_dirs() + conf = ctx.conf + if os.path.isdir(ctx.staging) and os.listdir(ctx.staging): + sys.exit( + "Staging folder {} is not empty, remove it first (a stale dump " + "must not be mixed with the new one)".format(ctx.staging) + ) + _makedirs(ctx.staging) + + prefixes = _prefixes(conf) + for query, prefix in zip(conf["queries"], prefixes): + command = [ + "inveniomigrator", + "dump", + "records", + "-q", + "{} {}".format(query.strip(), DUMP_EXCLUSIONS), + "--file-prefix", + prefix, + "--chunk-size={}".format(conf["chunk_size"]), + ] + print("Running: {}".format(" ".join(command))) + subprocess.check_call(command, cwd=ctx.staging) + + dumped = sorted( + name + for name in os.listdir(ctx.staging) + if any(name.startswith(prefix) for prefix in prefixes) + ) + if not dumped: + sys.exit("The dump produced no files in {}".format(ctx.staging)) + # dumped locally first, then copied checked: EOS can truncate a big write + for name in dumped: + copied = copy_verified( + os.path.join(ctx.staging, name), os.path.join(ctx.dump, name) + ) + print("{} {}".format("Copied" if copied else "Already there", name)) + shutil.rmtree(ctx.staging) + print("Dump placed in {} ({} file(s))".format(ctx.dump, len(dumped))) + + +# -------------------------------------------------------------------------- +# dump comments +# -------------------------------------------------------------------------- +def _export_comments_module(): + """Import subscripts/export_collection_comments.py (needs the legacy Invenio).""" + sys.path.insert(0, SUBSCRIPTS_DIR) + try: + import export_collection_comments + except ImportError as exc: + sys.exit( + "The legacy Invenio libraries are not available ({}): run this " + "command with the legacy interpreter, e.g. `ipython {} -- dump " + "comments`".format(exc, os.path.relpath(__file__)) + ) + return export_collection_comments + + +def cmd_dump_comments(ctx, args): + ctx.ensure_dirs() + export = _export_comments_module() + + recids = export.find_recids_with_comments(ctx.comments_queries()) + comments_metadata, users = export.dump_comments(recids) + if not comments_metadata: + print("No comments to migrate for this collection, nothing written.") + return + + # do not rename: the runners expect exactly these names + metadata_path = os.path.join(ctx.comments, "comments_metadata.json") + print("Creating {} file...".format(metadata_path)) + with open(metadata_path, "w") as fp: + json.dump(comments_metadata, fp) + + export.dump_users( + users, + os.path.join(ctx.comments_users, "active_commentors.json"), + os.path.join(ctx.comments_users, "missing_commentors_from_ldap.json"), + ) + + +# -------------------------------------------------------------------------- +# dump files +# -------------------------------------------------------------------------- +def cmd_dump_files(ctx, args): + ctx.ensure_dirs() + sys.path.insert(0, SUBSCRIPTS_DIR) + import copy_collection_files + + dump_files = sorted( + path + for path in copy_collection_files.get_dump_files_paths(ctx.dump) + if path.endswith(".json") + ) + if not dump_files: + sys.exit("No dump files in {}: run `dump records` first".format(ctx.dump)) + skip_plots = bool(ctx.conf["flags"].get("skip_plot_files")) + copy_collection_files.copy_collection_file( + dump_files, + ctx.files, + ctx.base, + skip_plots=skip_plots, + legacy_files_root=args.legacy_files_root, + ) + + # files attached to the comments, one folder per record + metadata_path = os.path.join(ctx.comments, "comments_metadata.json") + if not os.path.isfile(metadata_path): + print( + "No {}: comments attachments not copied (run `dump comments` " + "first if the collection has comments)".format(metadata_path) + ) + return + with open(metadata_path) as fp: + recids = json.load(fp).keys() + _export_comments_module().copy_attached_files(recids, ctx.comments) + + +# -------------------------------------------------------------------------- +# dump users +# -------------------------------------------------------------------------- +def cmd_dump_users(ctx, args): + """Dump the `people` authority records: input of `people-run`/people.csv.""" + _makedirs(ctx.users) + command = [ + "inveniomigrator", + "dump", + "records", + "-q", + '980__:"AUTHORITY" 980__:"PEOPLE"', + "--file-prefix", + "peoples", + "--chunk-size=1000", + ] + print("Running in {}: {}".format(ctx.users, " ".join(command))) + subprocess.check_call(command, cwd=ctx.users) + print( + "On the migration pod, if people.csv has to be recreated:\n" + " invenio migration users people-run --filepath {0}/people.csv " + "--dirpath {0}".format(ctx.users) + ) + + +# -------------------------------------------------------------------------- +# prepare +# -------------------------------------------------------------------------- +def cmd_prepare(ctx, args): + """Find the duplicates to redirect and check the configuration.""" + conf = ctx.conf + queries = conf["queries"] + analyze_query = conf.get("analyze_query") or ( + queries[0] if len(queries) == 1 else None + ) + if not analyze_query: + sys.exit( + "The collection has several queries: set `analyze_query` in {} " + "(one query, without -980:DELETED -980:HIDDEN -980__c:MIGRATED " + "-980__a:DUMMY)".format("collections/{}.yaml".format(conf["name"])) + ) + ctx.ensure_dirs() + + # the analysis writes duplicated_pids.json and ep_report_number_dups.json + # in the working directory of the collection + command = [ + args.python, + args.analyze_script, + "--collection-name", + conf["name"], + "--collection-q", + analyze_query, + "--output-dir", + ctx.base, + ] + print("Running: {}".format(" ".join(command))) + subprocess.check_call(command) + + print("Review ep_report_number_dups.json, then continue on the migration pod.") + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- +def build_parser(): + parser = argparse.ArgumentParser( + description=__doc__.split("\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--collection", help="collection name, as in collections/.yaml" + ) + common.add_argument( + "--confs-dir", + default=CONFS_DIR, + help="folder with the collection configurations (default: %(default)s)", + ) + common.add_argument( + "--env", help="override the environment read from the current directory" + ) + common.add_argument( + "--root", + help="root of the migration data, the folder holding users/ and the " + "collection folders (default: read from the EOS path, else the parent " + "of the current directory; env CDS_MIGRATION_ROOT)", + ) + + commands = parser.add_subparsers(dest="command") + commands.required = True + + dump = commands.add_parser("dump", help="dump records / comments / files / users") + targets = dump.add_subparsers(dest="target") + targets.required = True + for name, func, needs_conf, text in ( + ("records", cmd_dump_records, True, "dump the records of the collection"), + ("comments", cmd_dump_comments, True, "dump the comments and their commenters"), + ("files", cmd_dump_files, True, "copy record and comments files to EOS"), + ("users", cmd_dump_users, False, "dump the people authority records"), + ): + sub = targets.add_parser(name, parents=[common], help=text) + sub.set_defaults(func=func, needs_conf=needs_conf) + if name == "files": + sub.add_argument( + "--legacy-files-root", + default=LEGACY_FILES_ROOT, + help="where the legacy files are (default: %(default)s)", + ) + + prepare = commands.add_parser( + "prepare", parents=[common], help="find the duplicates to redirect" + ) + prepare.add_argument( + "--python", default="python2", help="interpreter of the analysis" + ) + prepare.add_argument("--analyze-script", default=ANALYZE_SCRIPT) + prepare.set_defaults(func=cmd_prepare, needs_conf=True) + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + args.func(Context(args, need_conf=args.needs_conf), args) + + +if __name__ == "__main__": + main() diff --git a/migration_cli/collections/_example.yaml b/migration_cli/collections/_example.yaml new file mode 100644 index 00000000..353eb26c --- /dev/null +++ b/migration_cli/collections/_example.yaml @@ -0,0 +1,43 @@ +# Configuration of one collection: copy this file to .yaml in this folder. +# It is the same on every environment. It is read by +# migration_cli/collection_migration.py on the legacy host and by the +# `invenio migration ... --collection ` commands on the pod, so there is no +# streams.yaml entry to add. The data (dump, files, comments) goes to +# //, see migrating-a-collection.md. +name: antares_restr +# folder: former_exp/antares_restr # working directory under the root (default: name) +filename_prefix: antares_restr # dump file prefix (default: name) +chunk_size: 1000 # records per dump file (default: 1000) + +# Legacy search queries WITHOUT the standard exclusions (-980:DELETED, ...): +# the script appends them. With several queries, each is dumped separately. +queries: + - '980__a:PRIVANTARES' + +# Needed with several queries: ONE query for the duplicates analysis, without +# -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY +# analyze_query: '980__a:PRIVANTARES OR 980__a:PRIVANTARESPLOT' + +flags: + restricted: true + plots: true + # create_inclusion_request: true + # update_new_version_publication_date: false # only for administrative docs + # preferred_model: research + # access_grants_view: false + # skip_plot_files: false # do not copy the files of type "Plot" + # workers: 4 # threads for the records transformation + +# Slugs of the communities the records are added to. The same in every +# environment: they are resolved to ids on the pod, and must exist there. +communities: + - cern-research # common community + - antares # the collection community + +# Queries to find the records whose comments are migrated (default: `queries`). +# comments_queries: +# - '980__a:PRIVANTARES' + +# Reviewers for the comments migration +# reviewers: +# - { group: antares-all } diff --git a/migration_cli/collections/ab_dep.yaml b/migration_cli/collections/ab_dep.yaml new file mode 100644 index 00000000..c72837c3 --- /dev/null +++ b/migration_cli/collections/ab_dep.yaml @@ -0,0 +1,10 @@ +name: ab_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 1c1519b8-8e2b-412c-999b-975291d0f5cf) + - cern-research diff --git a/migration_cli/collections/ac_dep.yaml b/migration_cli/collections/ac_dep.yaml new file mode 100644 index 00000000..04519db8 --- /dev/null +++ b/migration_cli/collections/ac_dep.yaml @@ -0,0 +1,10 @@ +name: ac_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was dfebd855-d156-4428-9b08-fe8ab8171799) + - cern-research diff --git a/migration_cli/collections/antares.yaml b/migration_cli/collections/antares.yaml new file mode 100644 index 00000000..867e9a7e --- /dev/null +++ b/migration_cli/collections/antares.yaml @@ -0,0 +1,9 @@ +name: antares +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - antares # TODO check the slug of the collection community (id e2d2ed70-8a3c-453b-aa5b-1fa985f0906b in the old streams) diff --git a/migration_cli/collections/antares_restr.yaml b/migration_cli/collections/antares_restr.yaml new file mode 100644 index 00000000..2c53f675 --- /dev/null +++ b/migration_cli/collections/antares_restr.yaml @@ -0,0 +1,9 @@ +name: antares_restr +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - antares # TODO check the slug of the collection community (id e2d2ed70-8a3c-453b-aa5b-1fa985f0906b in the old streams) diff --git a/migration_cli/collections/ar_dep.yaml b/migration_cli/collections/ar_dep.yaml new file mode 100644 index 00000000..e523611e --- /dev/null +++ b/migration_cli/collections/ar_dep.yaml @@ -0,0 +1,10 @@ +name: ar_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 2bf2998b-b310-4f36-9344-71e47560ae16) + - cern-research diff --git a/migration_cli/collections/at.yaml b/migration_cli/collections/at.yaml new file mode 100644 index 00000000..635800e9 --- /dev/null +++ b/migration_cli/collections/at.yaml @@ -0,0 +1,9 @@ +name: at +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + restricted: false + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 2ebd631f-66c3-4891-b2dd-275ec3656b7b) diff --git a/migration_cli/collections/dirac.yaml b/migration_cli/collections/dirac.yaml new file mode 100644 index 00000000..9c10c238 --- /dev/null +++ b/migration_cli/collections/dirac.yaml @@ -0,0 +1,9 @@ +name: dirac +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - dirac # slug of the community (id was 6a289642-5378-4daf-87b5-bb58af00487a) + - cern-research diff --git a/migration_cli/collections/drdc_comm.yaml b/migration_cli/collections/drdc_comm.yaml new file mode 100644 index 00000000..598413b5 --- /dev/null +++ b/migration_cli/collections/drdc_comm.yaml @@ -0,0 +1,7 @@ +name: drdc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 9eff4bab-9e5e-4cdb-9d16-be6588f7c6c1) diff --git a/migration_cli/collections/eec_comm.yaml b/migration_cli/collections/eec_comm.yaml new file mode 100644 index 00000000..ee02330e --- /dev/null +++ b/migration_cli/collections/eec_comm.yaml @@ -0,0 +1,7 @@ +name: eec_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 71a01272-00c7-48d2-9140-487d01ae8367) diff --git a/migration_cli/collections/emc_comm.yaml b/migration_cli/collections/emc_comm.yaml new file mode 100644 index 00000000..8bedcb76 --- /dev/null +++ b/migration_cli/collections/emc_comm.yaml @@ -0,0 +1,7 @@ +name: emc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 7b33dda1-a8b0-4d6b-808f-707fa3e63828) diff --git a/migration_cli/collections/est_dep.yaml b/migration_cli/collections/est_dep.yaml new file mode 100644 index 00000000..d71cc1ab --- /dev/null +++ b/migration_cli/collections/est_dep.yaml @@ -0,0 +1,8 @@ +name: est_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was d9c138bd-d39c-4797-a06c-c6f8db45f3ce) diff --git a/migration_cli/collections/imxgam.yaml b/migration_cli/collections/imxgam.yaml new file mode 100644 index 00000000..fe47d56f --- /dev/null +++ b/migration_cli/collections/imxgam.yaml @@ -0,0 +1,11 @@ +name: imxgam +folder: imXgam +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + restricted: true + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - imxgam # slug of the community (id was 88a5cdf4-974a-44d4-b145-d19ee6346bb8) + - cern-research diff --git a/migration_cli/collections/isc_comm.yaml b/migration_cli/collections/isc_comm.yaml new file mode 100644 index 00000000..468d9994 --- /dev/null +++ b/migration_cli/collections/isc_comm.yaml @@ -0,0 +1,7 @@ +name: isc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 12019bcd-28f2-40ce-85f3-3bfff399e847) diff --git a/migration_cli/collections/isolde.yaml b/migration_cli/collections/isolde.yaml new file mode 100644 index 00000000..07464dd5 --- /dev/null +++ b/migration_cli/collections/isolde.yaml @@ -0,0 +1,10 @@ +name: isolde +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true + preferred_model: isolde +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 1c46ceab-af68-4502-a436-cdf5a4cfbbe0) diff --git a/migration_cli/collections/isr_dep.yaml b/migration_cli/collections/isr_dep.yaml new file mode 100644 index 00000000..92b2ffdf --- /dev/null +++ b/migration_cli/collections/isr_dep.yaml @@ -0,0 +1,10 @@ +name: isr_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was c2fdb4ec-201c-4ec7-a9f0-23e4380a3fce) + - cern-research diff --git a/migration_cli/collections/isrc_comm.yaml b/migration_cli/collections/isrc_comm.yaml new file mode 100644 index 00000000..708d4ccd --- /dev/null +++ b/migration_cli/collections/isrc_comm.yaml @@ -0,0 +1,7 @@ +name: isrc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 2285af28-1808-4f11-aefb-64a5699940bb) diff --git a/migration_cli/collections/istc_comm.yaml b/migration_cli/collections/istc_comm.yaml new file mode 100644 index 00000000..3b570ca3 --- /dev/null +++ b/migration_cli/collections/istc_comm.yaml @@ -0,0 +1,7 @@ +name: istc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 84c28df7-fbee-4332-883b-7b9b77480e1f) diff --git a/migration_cli/collections/lep_dep.yaml b/migration_cli/collections/lep_dep.yaml new file mode 100644 index 00000000..34e6d9c5 --- /dev/null +++ b/migration_cli/collections/lep_dep.yaml @@ -0,0 +1,10 @@ +name: lep_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was f49a960d-94bf-432c-9ce6-c2fb9115ff9a) + - cern-research diff --git a/migration_cli/collections/lepc_comm.yaml b/migration_cli/collections/lepc_comm.yaml new file mode 100644 index 00000000..625d36c5 --- /dev/null +++ b/migration_cli/collections/lepc_comm.yaml @@ -0,0 +1,7 @@ +name: lepc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was a276bb34-e6ba-47c8-aa18-0ade1eec8b1d) diff --git a/migration_cli/collections/mps_div.yaml b/migration_cli/collections/mps_div.yaml new file mode 100644 index 00000000..64bedbcf --- /dev/null +++ b/migration_cli/collections/mps_div.yaml @@ -0,0 +1,10 @@ +name: mps_div +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 510206fb-2f80-44ee-8944-57f5dcfae5f0) + - cern-research diff --git a/migration_cli/collections/msc_div.yaml b/migration_cli/collections/msc_div.yaml new file mode 100644 index 00000000..3d0aa2cd --- /dev/null +++ b/migration_cli/collections/msc_div.yaml @@ -0,0 +1,10 @@ +name: msc_div +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 7bd98561-4985-434a-aed1-74301f865c24) + - cern-research diff --git a/migration_cli/collections/mt_dep.yaml b/migration_cli/collections/mt_dep.yaml new file mode 100644 index 00000000..eadee923 --- /dev/null +++ b/migration_cli/collections/mt_dep.yaml @@ -0,0 +1,8 @@ +name: mt_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was ad4469fa-5a17-41b9-825f-8010c379ba23) diff --git a/migration_cli/collections/na61.yaml b/migration_cli/collections/na61.yaml new file mode 100644 index 00000000..0561010f --- /dev/null +++ b/migration_cli/collections/na61.yaml @@ -0,0 +1,9 @@ +name: na61 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community diff --git a/migration_cli/collections/na62.yaml b/migration_cli/collections/na62.yaml new file mode 100644 index 00000000..505c74e1 --- /dev/null +++ b/migration_cli/collections/na62.yaml @@ -0,0 +1,9 @@ +name: na62 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community diff --git a/migration_cli/collections/na63.yaml b/migration_cli/collections/na63.yaml new file mode 100644 index 00000000..db3718c6 --- /dev/null +++ b/migration_cli/collections/na63.yaml @@ -0,0 +1,9 @@ +name: na63 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community diff --git a/migration_cli/collections/na64.yaml b/migration_cli/collections/na64.yaml new file mode 100644 index 00000000..40607d7f --- /dev/null +++ b/migration_cli/collections/na64.yaml @@ -0,0 +1,9 @@ +name: na64 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community diff --git a/migration_cli/collections/na65.yaml b/migration_cli/collections/na65.yaml new file mode 100644 index 00000000..e3ed6f03 --- /dev/null +++ b/migration_cli/collections/na65.yaml @@ -0,0 +1,9 @@ +name: na65 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community diff --git a/migration_cli/collections/na66.yaml b/migration_cli/collections/na66.yaml new file mode 100644 index 00000000..a924b469 --- /dev/null +++ b/migration_cli/collections/na66.yaml @@ -0,0 +1,9 @@ +name: na66 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community diff --git a/migration_cli/collections/nprc_comm.yaml b/migration_cli/collections/nprc_comm.yaml new file mode 100644 index 00000000..d4ffdfc3 --- /dev/null +++ b/migration_cli/collections/nprc_comm.yaml @@ -0,0 +1,7 @@ +name: nprc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 5c7c772d-8513-49d3-bd43-ac9a8beec02d) diff --git a/migration_cli/collections/nsc_comm.yaml b/migration_cli/collections/nsc_comm.yaml new file mode 100644 index 00000000..4e489b73 --- /dev/null +++ b/migration_cli/collections/nsc_comm.yaml @@ -0,0 +1,7 @@ +name: nsc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 847936da-b488-4426-b9aa-3659fa473779) diff --git a/migration_cli/collections/phi3_comm.yaml b/migration_cli/collections/phi3_comm.yaml new file mode 100644 index 00000000..7c94feef --- /dev/null +++ b/migration_cli/collections/phi3_comm.yaml @@ -0,0 +1,7 @@ +name: phi3_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was c35d2da3-8a47-420b-879e-be87b8b602c7) diff --git a/migration_cli/collections/phi_comm.yaml b/migration_cli/collections/phi_comm.yaml new file mode 100644 index 00000000..e1219848 --- /dev/null +++ b/migration_cli/collections/phi_comm.yaml @@ -0,0 +1,7 @@ +name: phi_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 2d9b3d0c-930a-4231-8c3e-55feaf9150c0) diff --git a/migration_cli/collections/ps_dep.yaml b/migration_cli/collections/ps_dep.yaml new file mode 100644 index 00000000..603bbaab --- /dev/null +++ b/migration_cli/collections/ps_dep.yaml @@ -0,0 +1,10 @@ +name: ps_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 5b634108-c93c-4740-af5c-31fb32e548eb) + - cern-research diff --git a/migration_cli/collections/psc_comm.yaml b/migration_cli/collections/psc_comm.yaml new file mode 100644 index 00000000..1436cde9 --- /dev/null +++ b/migration_cli/collections/psc_comm.yaml @@ -0,0 +1,7 @@ +name: psc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 9d91ff75-573c-48db-a1c1-4f78b0a9dced) diff --git a/migration_cli/collections/pscc_comm.yaml b/migration_cli/collections/pscc_comm.yaml new file mode 100644 index 00000000..1f3da9dd --- /dev/null +++ b/migration_cli/collections/pscc_comm.yaml @@ -0,0 +1,7 @@ +name: pscc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was dd2c1204-4508-43d6-a7ac-7ec5e1394523) diff --git a/migration_cli/collections/re29.yaml b/migration_cli/collections/re29.yaml new file mode 100644 index 00000000..10717def --- /dev/null +++ b/migration_cli/collections/re29.yaml @@ -0,0 +1,10 @@ +name: re29 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - re29 # slug of the community (id was aff67e22-9027-4401-a16f-1aaac2cd2c27) + - cern-research diff --git a/migration_cli/collections/sb_dep.yaml b/migration_cli/collections/sb_dep.yaml new file mode 100644 index 00000000..1e4a26be --- /dev/null +++ b/migration_cli/collections/sb_dep.yaml @@ -0,0 +1,8 @@ +name: sb_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was ca71c1ab-2eb3-4a43-a593-b8b140fe6123) diff --git a/migration_cli/collections/sc_dep.yaml b/migration_cli/collections/sc_dep.yaml new file mode 100644 index 00000000..eb48c106 --- /dev/null +++ b/migration_cli/collections/sc_dep.yaml @@ -0,0 +1,10 @@ +name: sc_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 369f4c3d-10fc-41e3-8cce-681fad5fcafe) + - cern-research diff --git a/migration_cli/collections/scc_comm.yaml b/migration_cli/collections/scc_comm.yaml new file mode 100644 index 00000000..1b4ed072 --- /dev/null +++ b/migration_cli/collections/scc_comm.yaml @@ -0,0 +1,7 @@ +name: scc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 28bf99b9-1e72-405a-b95a-828019831def) diff --git a/migration_cli/collections/scp_adv.yaml b/migration_cli/collections/scp_adv.yaml new file mode 100644 index 00000000..e65b5a9b --- /dev/null +++ b/migration_cli/collections/scp_adv.yaml @@ -0,0 +1,7 @@ +name: scp_adv +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was dbcc1269-a481-4d40-8ef9-66305c831840) diff --git a/migration_cli/collections/si_dep.yaml b/migration_cli/collections/si_dep.yaml new file mode 100644 index 00000000..84bc6f24 --- /dev/null +++ b/migration_cli/collections/si_dep.yaml @@ -0,0 +1,10 @@ +name: si_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was db44a69b-babc-48e8-9c03-29436b5bff07) + - cern-research diff --git a/migration_cli/collections/sl_dep.yaml b/migration_cli/collections/sl_dep.yaml new file mode 100644 index 00000000..4cae379e --- /dev/null +++ b/migration_cli/collections/sl_dep.yaml @@ -0,0 +1,10 @@ +name: sl_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was bcc34ae6-3a02-44cd-b08d-316a9d3e27e6) + - cern-research diff --git a/migration_cli/collections/sps_dep.yaml b/migration_cli/collections/sps_dep.yaml new file mode 100644 index 00000000..2ae7e25b --- /dev/null +++ b/migration_cli/collections/sps_dep.yaml @@ -0,0 +1,10 @@ +name: sps_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + create_inclusion_request: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 4bdb0cb3-b5f0-4383-a621-44b6e287438a) + - cern-research diff --git a/migration_cli/collections/spsc_comm.yaml b/migration_cli/collections/spsc_comm.yaml new file mode 100644 index 00000000..275ae048 --- /dev/null +++ b/migration_cli/collections/spsc_comm.yaml @@ -0,0 +1,7 @@ +name: spsc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 0ce97700-d80e-4bbb-89c4-690be8f37817) diff --git a/migration_cli/collections/spslc_comm.yaml b/migration_cli/collections/spslc_comm.yaml new file mode 100644 index 00000000..d2ef2b97 --- /dev/null +++ b/migration_cli/collections/spslc_comm.yaml @@ -0,0 +1,7 @@ +name: spslc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 4db1a084-4280-401d-a260-54229944af1e) diff --git a/migration_cli/collections/tcc_comm.yaml b/migration_cli/collections/tcc_comm.yaml new file mode 100644 index 00000000..e7523e57 --- /dev/null +++ b/migration_cli/collections/tcc_comm.yaml @@ -0,0 +1,7 @@ +name: tcc_comm +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - cern-research + - TODO # slug of the community (local id was 2ca82423-148a-42b5-86ff-03336b961c8e) diff --git a/migration_cli/collections/ts_dep.yaml b/migration_cli/collections/ts_dep.yaml new file mode 100644 index 00000000..fa6da363 --- /dev/null +++ b/migration_cli/collections/ts_dep.yaml @@ -0,0 +1,8 @@ +name: ts_dep +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - TODO # slug of the community (local id was 97ffcddd-ee1f-4e21-ac7b-b78612dc81d8) diff --git a/migration_cli/collections/ua2.yaml b/migration_cli/collections/ua2.yaml new file mode 100644 index 00000000..0721d112 --- /dev/null +++ b/migration_cli/collections/ua2.yaml @@ -0,0 +1,10 @@ +name: ua2 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true + preferred_model: research +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - ua2 # slug of the community (id was 4040d2c7-7648-4f1a-81fa-7112fec8d130) + - cern-research diff --git a/migration_cli/collections/ua4.yaml b/migration_cli/collections/ua4.yaml new file mode 100644 index 00000000..2331c870 --- /dev/null +++ b/migration_cli/collections/ua4.yaml @@ -0,0 +1,9 @@ +name: ua4 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - ua4 # slug of the community (id was 1b9239a7-5938-44e3-8ea8-2c530627f659) + - cern-research diff --git a/migration_cli/collections/ua5.yaml b/migration_cli/collections/ua5.yaml new file mode 100644 index 00000000..a7c0277f --- /dev/null +++ b/migration_cli/collections/ua5.yaml @@ -0,0 +1,9 @@ +name: ua5 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - ua5 # slug of the community (id was 7996b881-2420-4791-ac17-00cf3c1e7196) + - cern-research diff --git a/migration_cli/collections/ua8.yaml b/migration_cli/collections/ua8.yaml new file mode 100644 index 00000000..a8b64b32 --- /dev/null +++ b/migration_cli/collections/ua8.yaml @@ -0,0 +1,9 @@ +name: ua8 +queries: [] # TODO legacy queries of the collection (used by the legacy script only) +flags: + plots: true +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - ua8 # slug of the community (id was 2dbeb56c-b4ab-4cad-8d82-5a1f90499bd7) + - cern-research diff --git a/migration_cli/migrating-a-collection.md b/migration_cli/migrating-a-collection.md new file mode 100644 index 00000000..f795f445 --- /dev/null +++ b/migration_cli/migrating-a-collection.md @@ -0,0 +1,153 @@ +# Migrating a collection + +Replace every `` with your collection name, and every `` with your environment (`dev`, `sandbox`, `prod`). + +Everything is driven by one file per collection, `migration_cli/collections/.yaml`, kept in git. It is the same on every environment: both the legacy script (`cds-migration-01`) and the migrator pod read it, so there is **no `streams.yaml` entry and no config map change** for a collection. A change of the file is a normal merge request and a deployment of the image. + +The data of the collection is kept apart, under the root of the migration data of the environment: + +``` +/eos/media/cds/cds-rdm//migration// + dump/ records dump (dump records) + comments/ comments, commenters, attachments (dump comments / files) + files/ record files (dump files) + duplicated_pids.json duplicates to redirect (prepare) + ep_report_number_dups.json EP report numbers to review (prepare) +/eos/media/cds/cds-rdm//migration/users/ people.csv, shared by all collections +``` + +## 1. Data analysis (before moving to environments) + +Prepare the data locally first, see https://digital-repositories.web.cern.ch/cds/cds.cern.ch/migration/ and note the results in the issue: + +- [ ] check the permissions references and submissions (Submissions tab of the reference spreadsheet) +- [ ] check whether the e-group with the SBI right is still used; if yes, contact them +- [ ] check the fields of the submission form for unknown fields +- [ ] check the submission elements for hidden restrictions (specific emails and such) +- [ ] check the already submitted records and which collections they belong to, and which migration rules are missing + +Community details (fill in with the community): + +| Role | e-group/person | +|---------|----------------| +| Reader | TODO | +| Curator | TODO | +| Manager | TODO | +| Owner | TODO | + +## 2. Write the collection configuration + +Copy `migration_cli/collections/_example.yaml` to `migration_cli/collections/.yaml` and fill it in: + +```yaml +name: +filename_prefix: +queries: + - '980__a:PRIVANTARES' +flags: + restricted: true + plots: true +communities: + - cern-research + - +``` + +- Write the queries **without** the standard exclusions: the script appends `-690C_:SCICOM -595:*ARCHIVE* -980:ANNOUNCEMENT -980:DELETED -980:HIDDEN -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:GREYBOOK`. `-980__c:MIGRATED` is not excluded on purpose, some records belong to several communities. +- With several queries, also set `analyze_query`: one query, without `-980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY` (duplicates are searched among them to be redirected to). +- `communities` are slugs, the same in every environment. The community must already exist in the environment where you migrate: the pod resolves the slugs to ids and stops if one does not exist. +- Optional: `folder` (working directory under the data root, default the name), `comments_queries` (default `queries`) and `reviewers` for the comments, and the `flags` (`update_new_version_publication_date: true` only for administrative documents, e.g. HR; `skip_plot_files`; `workers`; ...). + +Merge it and update the checkout on `cds-migration-01` (`/opt/cds-migrator-kit`) and the image of the migration pod before going further. + +## 3. Block the submissions + +- [ ] merge and deploy the PR that blocks the submissions of the collection + +## 4. Export from the legacy system (on `cds-migration-01`) + +The scripts are read from the cds-migrator-kit checkout on the machine, `/opt/cds-migrator-kit/migration_cli` (make sure it is up to date). Use an identity that can write to the EOS migration folder, and run from the migration folder of the environment (the environment is read from the current directory), or pass `--env `. + +```shell +kinit cdsrdmeos +cd /eos/media/cds/cds-rdm//migration +S=/opt/cds-migrator-kit/migration_cli + +python $S/collection_migration.py dump records --collection +python $S/collection_migration.py dump comments --collection +python $S/collection_migration.py dump files --collection +python $S/collection_migration.py prepare --collection +``` + +| Command | Result | +|---|---| +| `dump records` | `inveniomigrator dump records` for each query, the dump files in `dump/` | +| `dump comments` | `comments/comments_metadata.json` and `comments/users/{active,missing}_commentors*.json`. Needs the legacy Invenio: `ipython $S/collection_migration.py -- dump comments --collection ` | +| `dump files` | copies the files of the dumped records to `files/` (checksums verified, `files.log` written) and the files attached to comments to `comments//` | +| `prepare` | runs `analyze_collection_migration.py` from `subscripts/`. Duplicates (same title, description and file checksums, `981__a` merges, ignoring `CER` values) in `duplicated_pids.json`; EP report numbers shared with other records in `ep_report_number_dups.json` (review them) | + + +If `people.csv` needs to be recreated (once per environment, no `--collection` needed): + +```shell +cd /eos/media/cds/cds-rdm//migration +python /opt/cds-migrator-kit/migration_cli/collection_migration.py dump users # dumps the authority records to users/ +# on the migration pod +invenio migration users people-run --filepath /eos/media/cds/cds-rdm//migration/users/people.csv \ + --dirpath /eos/media/cds/cds-rdm//migration/users +``` + +All the commands are safe to re-run. + +## 5. Before the migration + +- [ ] make sure the pod runs the latest cds-migrator-kit (the collection configuration is part of the image) +- [ ] **TAKE THE DB BACKUP** +- [ ] (optional) add the community to `INVENIO_CDS_COMMITTEE_APPROVAL_COMMUNITIES` (invenio config map) if needed +- [ ] the pod needs `INVENIO_CDS_MIGRATOR_KIT_ENV` set to the environment (`dev`, `sandbox`, `prod`; `production` is accepted for `prod`) + +## 6. Migrate (on the migration pod) + +```shell +oc project cds-rdm-sandbox +oc rsh + +# accounts of the record submitters, then of the commenters +invenio migration users submitters run --collection +invenio migration users commenters run --collection # make sure they are indexed, not only in the DB + +# records, then comments +invenio migration run --collection +invenio migration comments run --collection +``` + +Every command accepts `--dry-run` (try it first). The comments runner can be re-run after errors: it skips the comments already created. + +## 7. Statistics + +Raw events of the migrated collection (reads the records state file written by `migration run`, no path to give; `--filepath` overrides it): + +```shell +invenio migration stats events run --collection +``` + +When the events of **all** the collections are ingested, aggregate them once: + +```shell +invenio migration stats aggregate +``` + +It resets the `stats-bookmarks` index (with the index prefix of the environment), aggregates `record-view-agg` and `file-download-agg` up to today (`--end-date` to change) and reindexes `stats-record-view` and `stats-file-download`. + +## 8. After the migration + +- [ ] reindex all users `invenio rdm rebuild-all-indices -o users` to add missing migrated accounts +- [ ] set the community slug and deploy the redirections: [TODO PR with redirection] +- [ ] merge the Puppet `qa` branch to `master` to activate the submissions/collections/search redirections +- [ ] run the MIGRATED tagging on old CDS: https://gitlab.cern.ch/cds-team/production_scripts/-/blob/master/cds/set_records_migrated_to_rdm.py + +## Notes + +- **Data root**: the data goes to `/eos/media/cds/cds-rdm//migration` for `dev`, `sandbox` and `prod` (no setting needed). Locally (`INVENIO_CDS_MIGRATOR_KIT_ENV=local`, the default) the pod side uses `migration_cli/rdm-data` of the repository, or `INVENIO_CDS_MIGRATOR_KIT_MIGRATION_ROOT` (a relative path is relative to the repository root); the legacy script takes `--root` (or `CDS_MIGRATION_ROOT`) together with `--env local`. `dump files` takes `--legacy-files-root` for the source of the legacy files. The dump steps still need the legacy tools (`inveniomigrator`, legacy Invenio). See `cds_migrator_kit/rdm/README.md` for the local flow. +- **Configuration folder**: `migration_cli/collections` of the checkout. On the pod it is set by `INVENIO_CDS_MIGRATOR_KIT_COLLECTIONS_DIR` (done in the `Dockerfile`), on the legacy script by `--confs-dir`. +- Collections that still have an entry in `cds_migrator_kit/rdm/streams.yaml` and no configuration file in `migration_cli/collections/` keep working as before: the CLI falls back to `streams.yaml` if there is no `migration_cli/collections/.yaml`. +- `people.csv` is read from `/eos/media/cds/cds-rdm//migration/users/`; it is not copied into the collection. diff --git a/cds_migrator_kit/rdm/data/.gitkeep b/migration_cli/rdm-data/.gitkeep similarity index 100% rename from cds_migrator_kit/rdm/data/.gitkeep rename to migration_cli/rdm-data/.gitkeep diff --git a/migration_cli/step_by_step_documentation.md b/migration_cli/step_by_step_documentation.md new file mode 100644 index 00000000..1c1565a3 --- /dev/null +++ b/migration_cli/step_by_step_documentation.md @@ -0,0 +1,295 @@ +# Step by step documentation + +The detailed, manual description of each migration stage, kept as a reference. +The migration of a collection is described in `migrating-a-collection.md` +(`migration_cli/collection_migration.py` covers most of the stages below), and +the local flow in `cds_migrator_kit/rdm/README.md`. + +## Local analysis + +This is a detailed information on each stage of migration, but now they were all covered by migration_cli/collection_migration.py + +### To visualise the errors (locally): + +```shell +gunicorn -b :8080 --timeout 120 --graceful-timeout 60 cds_migrator_kit.app:app --reload --log-level debug --capture-output --access-logfile '-' --error-logfile '-' +``` + +### Legacy + +This is the recipe on how to dump the metadata and files. For local tests it is not necessary to do it often, especially for files which are static. +It makes sense to dump metadata if any changes are applied on legacy + +```shell +ssh cds-migration-01 # inveniomigrator tool installed here +cd /tmp/current-collection-dump +inveniomigrator dump records -q '037__:CERN-STUDENTS-Note-* -980:DELETED' --file-prefix summer-studends-notes --latest-only --chunk-size=1000 + +cd /eos/media/cds/cds-rdm/dev/migration/summer-student-notes/dump +kinit cdsrdmeosdev +cp /tmp/current-collection-dump/* . + + +# copy over all files to temporary location +ipython +# run script available in migration_cli/subscripts/copy_collection_files.py +# destination_prefix = "/eos/media/cds/cds-rdm/dev/migration/summer-student-notes/files" +# working_dir = "/eos/media/cds/cds-rdm/dev/migration/summer-student-notes" +# json_dump_dir = "/eos/media/cds/cds-rdm/dev/migration/summer-student-notes/dump" +``` + +Identify and dump duplicated records i.e. records with same title, description and file +checksums (excluding deleted files) on EOS. If duplicates found, then we keep the recid +that has restricted files or if not any, then the first one from the list. + +Run `migration_cli/collection_migration.py prepare` (it runs `migration_cli/subscripts/analyze_collection_migration.py`). + +#### How to mount eos locally on MAC (to copy over the dumps and files) + +1. Go to Finder icon on your dock +2. right click to get the contextual menu +3. choose connect to server +4. type `https://cernbox.cern.ch/cernbox/webdav/eos/media/cds/cds-rdm/dev/migration` +5. click connect +6. use eos account dev credentials + +### Openshift migration pod + +#### Collect and dump affiliation mapping + +In order to collect all affiliations from the collection dump folder run the following +command pointing to the `cds_migrator_kit.rdm.data.summer_student_reports.dump` +folder: + +``` +invenio migration affiliations run --filepath /path/to/cds_migrator_kit/rdm/migration/data/summer_student_reports/dump +``` + +This will collect and check each affiliation against the ROR organization API, and store them in the `cds_rdm.legacy.models.CDSMigrationAffiliationMapping` table. + +The model is then used during record migration to normalize the affiliation content following the below principles +to map the legacy input to a normalized value: + +1. If curated affiliation that might or not have a ROR ID exists then this value is used. +2. A ROR exact match +3. A ROR not exact match with a level of confidence of >= 90%. This will also flag the + record for further curation to validate the value. +4. The legacy affiliation value, and flag the record. + +#### Community id dump + +Dump the community id to migration `streams.yaml`(if the slug exists then it will read and dump the id only): + +```shell +invenio migration community dump --slug 'sspn' --title 'Summer Student Project Notes' --filepath /path/to/cds_migrator_kit/rdm/migration/streams.yaml +``` + +#### Missing Users files + +Make sure to add the user files required to be able to run the migration. The files can be found in `/eos/media/cds/cds-rdm/dev/migration/users/`. They should be stored in path indicated in the streams.yaml, the value that corresponds to the `record.transform.missing_users` key. + +#### Records migration + +Run the below command to migrate records in the created community from before: + +```shell +invenio migration run +``` + +#### EP approval records + +EP approval records are detected automatically per-record (via the `9031_:EPPHAPP` history popped during transform) and split into a public/restricted pair by the loader - no separate flag or stream is needed, they can be migrated in the same run as regular records. + +1. **Dump** them filtered by `9031_:EPPHAPP`, for example (FASER): + +```bash +inveniomigrator dump records -q '980__a:ARTICLE or 980__a:PREPRINT and 693:"FASER" not 980:CONFERENCEPAPER not 591__b:"Draft" 9031_:EPPHAPP -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY' --file-prefix faser-papers-cern-ep --chunk-size=1000 +``` + +2. Configure a collection entry in `streams.yaml` pointing at that dump. + +3. **Migrate** normally (dry run first): + +```shell +invenio migration run --collection faser-ep --dry-run +invenio migration run --collection faser-ep +``` + +#### Comments migration + +Configure the collection under `comments` in `streams.yaml` (`dir_path`, `reviewers`). Paths and reviewers are loaded from there when you pass `--collection`. + +1. Dump comments for the collection and generate commenter metadata: + +```shell +# on cds-migration-01 +ipython ./migration_cli/collection_migration.py -- dump comments --collection +``` + +Output: `comments_metadata.json`. The script also generates `missing_commentors_from_ldap.json`. + +**Do not rename these files** — the runners expect exactly `comments_metadata.json` under `dir_path` and `missing_commentors_from_ldap.json` under `dir_path/users/`. + +2. Ensure files are in place (paths should match `streams.yaml`): + +- `comments_metadata.json` under the collection `dir_path` +- `people.csv` and `missing_commentors_from_ldap.json` under `dir_path/users/` + +```shell +cp /eos/media/cds/cds-rdm//migration/users/people.csv /eos/media/cds/cds-rdm//migration//comments/users/ +``` + +3. Pre-create commenter accounts (dry run first): + +```shell +# on migration pod +invenio migration comments commenters-run --collection --dry-run +invenio migration comments commenters-run --collection +``` + +After running, make sure the users are indexed and not only created in the DB. + +4. Migrate comments (dry run first): + +```shell +# on migration pod +invenio migration comments run --collection --dry-run +invenio migration comments run --collection +``` + +The comments runner can be re-run after errors; already migrated comments are skipped. + +### Migrate the statistics for the successfully migrated records + +When the `invenio migration run` command ends it will produce a `rdm_records_state.json` file which has linked information about the migrated records and the old system. The format will be similar to below: + +```json +{ + "legacy_recid": "2884810", + "parent_recid": "zts3q-6ef46", + "parent_object_uuid": "155be22f-3038-49e0-9f17-9518eaac783a", + "latest_version": "1mae4-skq89", + "latest_version_object_uuid": "155be22f-3038-49e0-9f17-9518eaac783a", + "versions": [ + { + "new_recid": "1mae4-skq89", + "version": 2, + "files": [ + { + "legacy_file_id": 1568736, + "bucket_id": "155be22f-3038-49e0-9f17-9518eaac783a", + "file_key": "Summer student program report.pdf", + "file_id": "06cdb9d2-635f-4dbe-89fe-4b27afddeaa2", + "size": "1690854" + } + ] + } + ] +} +``` + +- Open the `cds_migrator_kit/rdm/migration/stats/config.py` and + + - export the below 2 environmental variables + - `CDS_MIGRATOR_KIT_SRC_SEARCH_HOSTS`: e.g `export CDS_MIGRATOR_KIT_SRC_SEARCH_HOSTS='[{"host": "os-cds-legacy.cern.ch", "url_prefix": "/os", "timeout": 30, "port": 443, "use_ssl": true, "verify_certs": false, "http_auth": ["LEGACY_PRODUCTION_OPENSEARCH_USERNAME", ""]}]'` + - you find the credentials for `LEGACY_PRODUCTION_OPENSEARCH_PASSWORD` by `tbag show LEGACY_PRODUCTION_OPENSEARCH_PASSWORD --hg cds` + - you find the credentials for `LEGACY_PRODUCTION_OPENSEARCH_USERNAME` by `tbag show LEGACY_PRODUCTION_OPENSEARCH_USERNAME --hg cds` + +- Open a shell and run the following commands + +```bash +$ invenio migration stats run --filepath "path/to/file/of/rdm_records_state.json" +``` + +This will migrate only the raw statistic events. When all events are ingested to the new cluster then we will need to aggregate them. + +To do so, you need to run after you have set the correct bookmark for each event: + +on opensearch + +```shell +DELETE /cds-rdm-stats-bookmarks + +POST /cds-rdm-stats-bookmarks/_doc +{ + "date": "2000-06-26T15:56:05.755394", + "aggregation_type": "file-download-agg" +} +POST /cds-rdm-stats-bookmarks/_doc +{ + "date": "2000-06-26T15:56:05.755394", + "aggregation_type": "record-view-agg" +} +POST /cds-rdm-stats-bookmarks/_doc +{ + "date": "2000-06-26T15:56:05.755394", + "aggregation_type": "stats_reindex" +} +``` + +``` +from invenio_stats.tasks import aggregate_events + +start_date = '2000-01-01' +end_date = '2024-12-01' + +aggregations = ["record-view-agg", "file-download-agg"] +aggregate_events(aggregations) + +from invenio_rdm_records.services.tasks import reindex_stats +stats_indices = [ "stats-record-view", "stats-file-download",] + +reindex_stats(stats_indices) +``` + +visit https://migration-cds-rdm-dev.app.cern.ch for report + +## Local rerun migration from clean state without setup everything again + +If you want to cleanup a previous migration run without having to re setup everything +i.e not repopulating all vocabularies which takes a lot of time, then run the following +recipe: + +- Cleanup db tables from pgadmin (**!!! ONLY ON DEV or local !!!**) + +```sql +DELETE FROM rdm_versions_state; +DELETE FROM rdm_records_files; +DELETE FROM rdm_drafts_files; +DELETE FROM rdm_records_metadata; +DELETE FROM rdm_drafts_metadata; +DELETE FROM rdm_parents_metadata; +DELETE FROM communities_files; +DELETE FROM oaiserver_set; +DELETE FROM communities_metadata; +DELETE FROM files_objecttags; +DELETE FROM files_object; +DELETE FROM files_buckettags; +DELETE FROM files_bucket; +DELETE FROM files_files; +DELETE FROM pidstore_pid WHERE pid_type = 'lrecid'; +DELETE FROM pidstore_pid WHERE pid_type = 'recid'; +DELETE FROM pidstore_pid WHERE pid_type = 'doi'; +DELETE FROM cds_clc_record_sync; +DELETE FROM cds_migration_legacy_records; +``` + +- Cleanup indexed documents from opensearch + +``` +POST /cds-rdm-dev-rdmrecords/_delete_by_query +{ + "query": { + "match_all": {} + } +} +POST /cds-rdm-dev-communities/_delete_by_query +{ + "query": { + "match_all": {} + } +} +``` + +- Rerun `invenio migration community dump --slug 'sspn' --title 'Summer Student Project Notes' --filepath /path/to/cds_migrator_kit/rdm/migration/streams.yaml` +- Rerun `invenio migration run` diff --git a/scripts/analyze_collection_migration.py b/migration_cli/subscripts/analyze_collection_migration.py similarity index 93% rename from scripts/analyze_collection_migration.py rename to migration_cli/subscripts/analyze_collection_migration.py index a955d020..87e63d17 100644 --- a/scripts/analyze_collection_migration.py +++ b/migration_cli/subscripts/analyze_collection_migration.py @@ -7,15 +7,18 @@ 1. Connect to the CDS-legacy migration node 2. Run this script with --collection-name and --collection-q. 3. Duplicate recids and 981__a merges are written to - /tmp//legacy_pids_to_redirect.json + /duplicated_pids.json 4. EP report numbers shared with other CDS records are written to - /tmp//ep_report_number_dups.json + /ep_report_number_dups.json + + is --output-dir: the collection working directory on EOS, the one +with its migration_conf.yaml (`collection_migration.py prepare` passes it). Do not use -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY in the query. Example: - python2 scripts/analyze_collection_migration.py \\ + python2 migration_cli/subscripts/analyze_collection_migration.py \\ --collection-name faser \\ --collection-q '980__a:ARTICLE or 980__a:PREPRINT and 693:"FASER"' """ @@ -296,7 +299,7 @@ def parse_args(): parser.add_argument( "--collection-name", required=True, - help="Collection name used for the dump directory (/tmp/).", + help="Collection name, only used in the output messages.", ) parser.add_argument( "--collection-q", @@ -306,6 +309,11 @@ def parse_args(): "-980:HIDDEN -980__c:MIGRATED -980__a:DUMMY." ), ) + parser.add_argument( + "--output-dir", + required=True, + help="Collection working directory the results are written to.", + ) return parser.parse_args() @@ -314,7 +322,7 @@ def main(): collection_name = args.collection_name collection_q = args.collection_q - json_dump_dir = "/tmp/{}".format(collection_name) + json_dump_dir = args.output_dir if not os.path.exists(json_dump_dir): os.makedirs(json_dump_dir) @@ -327,14 +335,10 @@ def main(): redirect_map = search_dups(collection_q) redirect_map.update(search_merges(collection_q)) - with open("{}/legacy_pids_to_redirect.json".format(json_dump_dir), "w+") as fp: + redirect_path = os.path.join(json_dump_dir, "duplicated_pids.json") + with open(redirect_path, "w+") as fp: json.dump(redirect_map, fp) - _print_green( - "Wrote {} duplicates to {}".format( - len(redirect_map), - "{}/legacy_pids_to_redirect.json".format(json_dump_dir), - ) - ) + _print_green("Wrote {} duplicates to {}".format(len(redirect_map), redirect_path)) _print_green("--------------------------------") _print_green("Searching for EP report number duplicates...") diff --git a/scripts/copy_collection_files.py b/migration_cli/subscripts/copy_collection_files.py similarity index 70% rename from scripts/copy_collection_files.py rename to migration_cli/subscripts/copy_collection_files.py index 631862eb..9289ee25 100644 --- a/scripts/copy_collection_files.py +++ b/migration_cli/subscripts/copy_collection_files.py @@ -17,7 +17,13 @@ def file_md5(path): return md5.hexdigest() -def copy_collection_file(dump_files, destination_prefix, working_dir, skip_plots=False): +def copy_collection_file( + dump_files, + destination_prefix, + working_dir, + skip_plots=False, + legacy_files_root="/opt/cdsweb/var/data/files/", +): file_log = open(os.path.join(working_dir, "files.log"), "wb") for i, dump_file in enumerate(dump_files): @@ -38,7 +44,7 @@ def copy_collection_file(dump_files, destination_prefix, working_dir, skip_plots continue full_path = legacy_record_file["full_path"] # important: last slash - path_to_replace = "/opt/cdsweb/var/data/files/" + path_to_replace = legacy_files_root rel_path = full_path.replace(path_to_replace, "") destination_path = os.path.join(destination_prefix, rel_path) @@ -83,17 +89,38 @@ def get_dump_files_paths(json_dump_dir): return dump_files -collection = "lep/aleph_drafts" -environment = "dev" -skip_plots = False +def main(): + parser = argparse.ArgumentParser( + description="Copy the files of a dumped collection to EOS." + ) + parser.add_argument( + "--collection", required=True, help="collection folder, e.g. small_exp/ua8" + ) + parser.add_argument("--environment", required=True, help="dev / sandbox / prod") + parser.add_argument( + "--root", + default="/eos/media/cds/cds-rdm/{env}/migration", + help="migration data root, {env} is replaced by --environment", + ) + parser.add_argument( + "--skip-plots", + required=True, + choices=["true", "false"], + help="do not copy the files of type Plot", + ) + args = parser.parse_args() -destination_prefix = "/eos/media/cds/cds-rdm/{0}/migration/{1}/files".format( - environment, collection -) -working_dir = "/eos/media/cds/cds-rdm/{0}/migration/{1}".format(environment, collection) -json_dump_dir = "/eos/media/cds/cds-rdm/{0}/migration/{1}/dump".format( - environment, collection -) + base = os.path.join( + args.root.format(env=args.environment).rstrip("/"), args.collection + ) + dump_files = get_dump_files_paths(os.path.join(base, "dump")) + copy_collection_file( + dump_files, + os.path.join(base, "files"), + base, + skip_plots=args.skip_plots == "true", + ) -dump_files = get_dump_files_paths(json_dump_dir) -copy_collection_file(dump_files, destination_prefix, working_dir, skip_plots=skip_plots) + +if __name__ == "__main__": + main() diff --git a/scripts/export_collection_comments.py b/migration_cli/subscripts/export_collection_comments.py similarity index 91% rename from scripts/export_collection_comments.py rename to migration_cli/subscripts/export_collection_comments.py index db294f9b..3e6829db 100644 --- a/scripts/export_collection_comments.py +++ b/migration_cli/subscripts/export_collection_comments.py @@ -1,20 +1,22 @@ """ One-shot export of a collection's legacy comments (run on cds-migration-01). -Consolidates, in this order: - 1. dump_comments_to_migrate.py -> comments_metadata.json - + active_commentors.json - + missing_commentors_from_ldap.json - 2. copying people.csv -> /users/people.csv - 3. copy_comments_attached_files.py -> //... +Its functions are used by migration_cli/collection_migration.py (`dump comments` +and `dump files`), which is the way to run the export. Run on its own, it does, +in this order: + 1. dump the comments -> comments_metadata.json + + active_commentors.json + + missing_commentors_from_ldap.json + 2. copy people.csv -> /users/people.csv + 3. copy the attachments -> //... Usage (Python 2 / ipython of the legacy Invenio, so no f-strings here): - ipython ./scripts/export_collection_comments.py -- \ + ipython ./migration_cli/subscripts/export_collection_comments.py -- \ --env dev --collection antares_restr --queries-file antares_queries.txt # or pass queries inline (repeatable) - ipython ./scripts/export_collection_comments.py -- \ + ipython ./migration_cli/subscripts/export_collection_comments.py -- \ --env dev --collection antares_restr --query '980__a:PRIVANTARES ...' The queries file has one legacy search query per line (blank lines and lines @@ -27,7 +29,9 @@ users/people.csv, users/active_commentors.json, users/missing_commentors_from_ldap.json -Afterwards, on the migration pod, run scripts/migrate_collection_comments.sh. +Afterwards, on the migration pod, run +`invenio migration users commenters run --collection ` and +`invenio migration comments run --collection `. The script is safe to re-run: files are overwritten and already copied attachment folders are skipped. @@ -155,7 +159,11 @@ def dump_comments(recids): users_metadata = set() for i, recid in enumerate(recids): - print("({}/{}) Processing comments for record<{}>".format(i + 1, len(recids), recid)) + print( + "({}/{}) Processing comments for record<{}>".format( + i + 1, len(recids), recid + ) + ) comments = get_comments_from_legacy(recid) if not comments: @@ -350,10 +358,10 @@ def main(): print("\nDone. Next, on the migration pod:") print( - " ./scripts/migrate_collection_comments.sh --collection {}".format( - args.collection - ) + " invenio migration users commenters run --collection {0}\n" + " invenio migration comments run --collection {0}".format(args.collection) ) -main() +if __name__ == "__main__": + main() diff --git a/scripts/copy_comments_attached_files.py b/scripts/copy_comments_attached_files.py deleted file mode 100644 index 37a323d1..00000000 --- a/scripts/copy_comments_attached_files.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -This script is used to copy the attached files from the old system to the migration folder in the new eos. - -1. Read the attached files from the old system -2. Copy the attached files to the migration folder in the new eos system -3. Save the attached files mapping to a json file for attaching them to the comments in the new system -""" - -# from invenio.webcomment import get_attached_files -import json -import os -import shutil - -collection = "it_meetings" -environment = "sandbox" - -source_prefix = "/opt/cdsweb/var/data/comments" -destination_prefix = "/eos/media/cds/cds-rdm/{0}/migration/{1}/comments".format( - environment, collection -) -""" -collection_name/ -|-- comments/ - |-- comments_metadata.json - |-- recid/ - |-- comment_id (or reply_comment_id)/ - |-- This is where the attached file is copied to - |-- users/ - |-- people.csv - |-- active_commentors.json - |-- missing_commentors_from_ldap.json - -(We keep the recid folder to avoid confusion with the files folder and in case different comments contain the same file name) -(We keep the comment_id (or reply_comment_id) folder to avoid confusion with the files folder and in case different comments contain the same file name) -""" - - -def copy_comments_attached_files(comments_metadata): - for recid in comments_metadata.keys(): - # Copy the whole /comments/{recid} folder to the destination folder - shutil.copytree( - os.path.join(source_prefix, recid), os.path.join(destination_prefix, recid) - ) - print("Copied {} comments directory to {}".format(recid, destination_prefix)) - - -# Load the comments metadata to get the recids with comments and the comment IDs -comments_metadata_json_file = os.path.join(destination_prefix, "comments_metadata.json") -with open(comments_metadata_json_file, "r") as f: - comments_metadata = json.load(f) - -copy_comments_attached_files(comments_metadata) diff --git a/scripts/dump_comments_to_migrate.py b/scripts/dump_comments_to_migrate.py deleted file mode 100644 index 00edb5ce..00000000 --- a/scripts/dump_comments_to_migrate.py +++ /dev/null @@ -1,382 +0,0 @@ -""" -This script is used to get the comments from the legacy system and save them to a json file. -It also dumps the users metadata into active_commentors.json and missing_commentors_from_ldap.json files. -""" - -""" -1. Find migrated records with comments using https://digital-repositories.web.cern.ch/cds/cds.cern.ch/migration/#verify-if-the-collection-has-commentsdiscussions -2. Extract and sanitize comments (query_retrieve_comments_or_remarks) -3. Map legacy user id to cdsrdm user id -4. Create comments metadata -5. Save comments metadata to json file -6. Dump the users metadata into active_commentors.json and missing_commentors_from_ldap.json files. -""" - -import json -import os - -from invenio.bibcirculation_cern_ldap import get_user_info_from_ldap -from invenio.dbquery import run_sql -from invenio.search_engine import search_pattern -from invenio.webcomment_dblayer import get_comment_to_bibdoc_relations - - -def get_comments_from_legacy(recid, display_order="cmt.date_creation ASC"): - """ - # from invenio.webcomment import query_retrieve_comments_or_remarks - """ - query = """SELECT user.nickname, - user.email, - user.note, - user.last_login, - cmt.id_user, - DATE_FORMAT(cmt.date_creation, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s') as created, - cmt.body, - cmt.status, - cmt.nb_abuse_reports, - cmt.id, - cmt.round_name, - cmt.restriction, - %(reply_to_column)s as reply_to_id, - cmt.body_format - FROM cmtRECORDCOMMENT cmt LEFT JOIN user ON - user.id=cmt.id_user - WHERE cmt.id_bibrec=%%s - ORDER BY %(display_order)s - """ % { - "display_order": display_order, - "reply_to_column": recid > 0 - and "cmt.in_reply_to_id_cmtRECORDCOMMENT" - or "cmt.in_reply_to_id_bskRECORDCOMMENT", - } - params = (recid,) - res = run_sql(query, params, with_dict=True) - return res - - -# Function to flatten arbitrarily nested comment replies into a 1-level replies list -def flatten_replies(comments_list): - """ - Takes a list of comments (dicts with at least id, reply_to_id), and - returns a list of top-level comments each with direct and indirect (flattened) - replies under a single 'replies' key (1-level nesting). - Each comment dict may have its own 'replies' key (list). - """ - # Build maps for quick lookup - comments_by_id = {c["id"]: dict(c, replies=[]) for c in comments_list} - top_level_comments = [] - - for c in comments_list: - parent = c.get("reply_to_id") - if parent is None or parent not in comments_by_id: - # This is a top-level comment - top_level_comments.append(comments_by_id[c["id"]]) - else: - # This is a reply; add to parent's replies - comments_by_id[parent]["replies"].append(comments_by_id[c["id"]]) - - def collect_all_replies(comment): - """Recursively flattens all indirect replies under this comment.""" - flat = [] - queue = list(comment["replies"]) - while queue: - reply = queue.pop(0) - flat.append(reply) - # Add all replies to the end of the queue - queue.extend(reply["replies"]) - # Remove their nested replies again for full flatten - for r in flat: - r["replies"] = [] - # Sort the replies by creation date - flat.sort(key=lambda x: x["created"]) - comment["replies"] = flat - - for comment in top_level_comments: - collect_all_replies(comment) - - return top_level_comments - - -def find_recids_comments(queries): - recids_list = [] - print("Querying already migrated records...") - for i, collection_query in enumerate(queries): - recs = search_pattern(p=collection_query) - print( - "({} / {}) Found {} records for query `{}`".format( - i + 1, len(queries), len(recs), collection_query - ) - ) - recids_list.extend(recs) - - print("Filtering records with comments...") - records_with_comments = run_sql( - "SELECT id_bibrec " - "FROM cmtRECORDCOMMENT " - "WHERE id_bibrec IN ({}) " - "GROUP BY id_bibrec " - "HAVING COUNT(*) > 0".format(",".join([str(recid) for recid in recids_list])), - ) - - recids_with_comments = [record[0] for record in records_with_comments] - print( - "Found {} records with comments out of {} migrated records".format( - len(recids_with_comments), len(recids_list) - ) - ) - return recids_with_comments - - -def dump_comments_and_users(recids): - comments_metadata = {} - """ - { - recid: [ - { - comment_id: comment_id, - content: content, - status: status, - user_email: user_email, - created_at: created_at, - file_relation: {file_id: file_id, version: version}, - replies: [ - { - comment_id: reply_comment_id, - content: reply_comment_content, - status: reply_comment_status, - user_email: reply_comment_user_email, - created_at: reply_comment_created_at, - file_relation: {file_id: file_id, version: version}, - reply_to_id: id_of_the_comment_replied_to, - } - ] - } - ] - } - """ - - users_metadata = set() - """ - [(user_id, user_email, user_nickname, user_note, user_last_login), ...] - """ - - for i, recid in enumerate(recids): - print( - "({}/{}) Processing comments for record<{}>".format( - i + 1, len(recids), recid - ) - ) - comments = get_comments_from_legacy(recid) - - if not comments: - print("No comments found for record<{}>. Skipping...".format(recid)) - continue - - # Check if the comments list have atleast 1 comment with 'ok' status or 'approved' status to avoid migrating records with spam comments that are already deleted - # For eg.: https://cds.cern.ch/record/1367848/comments?ln=en - if not any(comment["status"] in ["ok", "ap"] for comment in comments): - print( - "No comments with atleast one 'ok'/'ap' status found for record<{}>. Skipping...".format( - recid - ) - ) - continue - - print("Found `{}` comment(s) for record<{}>".format(len(comments), recid)) - comments_metadata[recid] = [] - - # `get_comment_to_bibdoc_relations` is used to find if comments are attached to the record's files (and the version of the files) - # This is not the same as the files attached to the comments - comments_to_file_relations = get_comment_to_bibdoc_relations(recid) - comment_to_version_relations = {} - for relation in comments_to_file_relations: - comment_to_version_relations[relation["id_comment"]] = { - "file_id": relation["id_bibdoc"], - "version": relation["version"], - } - print( - "Found {} comments to file relations for record {} ...".format( - len(comments_to_file_relations), recid - ) - ) - - for comment in comments: - users_metadata.add( - ( - comment["id_user"], - comment["email"], - comment["nickname"], - comment["note"], - comment["last_login"], - ), - ) - - # Flatten the reply comments - flattened_comments = flatten_replies(comments) - # Sanitize the comment metadata for RDM - for comment in flattened_comments: - comment_data = { - "comment_id": comment["id"], - "content": comment["body"] - .replace("\xc2\xa0", " ") - .replace("\n", "") - .strip(), - "status": comment.get("status"), - "user_email": comment.get("email"), - "created_at": comment.get("created"), - "file_relation": comment_to_version_relations.get(comment["id"], {}), - "replies": [ - { - "comment_id": reply["id"], - "content": reply["body"] - .replace("\xc2\xa0", " ") - .replace("\n", "") - .strip(), - "status": reply.get("status"), - "user_email": reply.get("email"), - "created_at": reply.get("created"), - "reply_to_id": reply.get("reply_to_id"), - "file_relation": comment_to_version_relations.get(reply["id"], - {}), - } - for reply in comment["replies"] - ], - } - comments_metadata[recid].append(comment_data) - print("Successfully processed comment(s) for record<{}>!!!".format(recid)) - - print("Creating {0} file...".format(COMMENTS_METADATA_FILEPATH)) - with open(COMMENTS_METADATA_FILEPATH, "w") as f: - json.dump(comments_metadata, f) - # This file will be read and run by the CommentsRunner to migrate the comments. - return comments_metadata, users_metadata - - -""" -The following snippet is taken from the `dump_users.py` script in the `production_scripts` repository: -https://gitlab.cern.ch/cds-team/production_scripts/-/blob/master/cds-rdm/migration/dump_users.py?ref_type=heads - -It is used to dump the users metadata as active_commentors.json and missing_commentors_from_ldap.json files. - -After running this script, place the "active_commentors.json" and "missing_commentors_from_ldap.json" files in the "cds_migrator_kit/rdm/data/users/" folder along with "people.csv" file. -""" - - -def dump_users(users): - """ - Dump the users metadata into active_commentors.json and missing_commentors_from_ldap.json files. - """ - - def get_uid_from_ldap_user(ldap_user): - try: - uidNumber = ldap_user["uidNumber"][0] - return uidNumber - except: - return None - - def get_department_from_ldap_user(ldap_user): - try: - department = ldap_user["department"][0] - return department - except: - return None - - def _dump(recs): - valid_users = [] - missing_users = [] - - for rec in recs: - # record example: ( - # 414320 - # joe.doe@mail.com - # ) - - email = rec[1] - ldap_user = get_user_info_from_ldap(email=email) - uidNumber = get_uid_from_ldap_user(ldap_user) - record = { - "id": rec[0], - "email": rec[1], - "displayname": rec[2], - "active": rec[3], - } - if uidNumber: - record["uid"] = uidNumber - department = get_department_from_ldap_user(ldap_user) - if department: - record["department"] = department - else: - print("No department for {}".format(email)) - valid_users.append(record) - else: - missing_users.append(record) - - return valid_users, missing_users - - valid_users, missing_users = _dump(users) - with open(USERS_FILEPATH, "w") as fp: - json.dump(valid_users, fp, indent=2) - - if missing_users: - print( - "Missing users found {0}. Creating {1} file...".format( - len(missing_users), MISSING_USERS_FILEPATH - ) - ) - with open(MISSING_USERS_FILEPATH, "w") as fp: - json.dump(missing_users, fp, indent=2) - - -ENV = "dev" -COLLECTION = "it_dep" -BASE_OUTPUT_DIR = "/eos/media/cds/cds-rdm/{0}/migration/{1}/".format(ENV, COLLECTION) -COMMENTS_METADATA_DIR = os.path.join(BASE_OUTPUT_DIR, "comments") -if not os.path.exists(COMMENTS_METADATA_DIR): - os.makedirs(COMMENTS_METADATA_DIR) -COMMENTS_METADATA_FILEPATH = os.path.join( - COMMENTS_METADATA_DIR, "comments_metadata.json" -) - -OUTPUT_DIR = os.path.join(COMMENTS_METADATA_DIR, "users") -USERS_FILEPATH = os.path.join(OUTPUT_DIR, "active_commentors.json") -MISSING_USERS_FILEPATH = os.path.join(OUTPUT_DIR, "missing_commentors_from_ldap.json") - -if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) - -collection_queries = [ - '980:L3_Papers -690C_:SCICOM -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:ANNOUNCEMENT -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:INTNOTEALICEPUBL', - '693__e:L3 -980:GREYBOOK -980:Press_Release -980:INTNOTECMSPUBL -980:ANNOUNCEMENT -690C_:SCICOM -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:PHOTOLAB -980:PHOTOARC -980:VIDEOARC -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:INTNOTEALICEPUBL', - '980:L3_Misc -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:INTNOTEALICEPUBL', - - '693__e:OPAL -980:GREYBOOK -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -980:PHOTOLAB -980:PHOTOARC -980:VIDEOARC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '980:OPAL_Papers -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLSPSC -980:SCICOMMPUBLLEPC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '980:OPAL_Misc -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLSPSC -980:SCICOMMPUBLLEPC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '980:DELPHI_Misc -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLSPSC -980:SCICOMMPUBLLEPC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '980:DELPHI_Papers -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLSPSC -980:SCICOMMPUBLLEPC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '690:PUBLDELPHINOTE -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLSPSC -980:SCICOMMPUBLLEPC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '690:PRIVDELPHINOTE -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLSPSC -980:SCICOMMPUBLLEPC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '980:INTNOTEALEPHPRIV -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLSPSC -980:SCICOMMPUBLLEPC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:ALEPHDRAFT', - - '037:CERN-ALEPH-PUB* -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:ALEPHDRAFT', - - '710:"ALEPH Collaboration" -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:ALEPHDRAFT', - - '980:ALEPHDRAFT -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*', - - '037:CERN-ALEPH-ARCH-DATA* -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-* -980:ALEPHDRAFT', - - '693__e:DELPHI -690:PRIVDELPHINOTE -980:GREYBOOK -980:LHCb_Misc -690C_:SCICOM -980:ANNOUNCEMENT -980:SCICOMMPUBLLEPC -980:SCICOMMPUBLSPSC -980:PHOTOLAB -980:PHOTOARC -980:VIDEOARC -595:PSARCHIVE -595:SPSARCHIVE -595:LEPARCHIVE -980:DELETED -980:HIDDEN -980__c:MIGRATED -980__a:DUMMY -690C_:movingimages -970__b:MERGE-*' - -] - -recids_with_comments = find_recids_comments(collection_queries) -comments, users = dump_comments_and_users(recids_with_comments) -dump_users(users) diff --git a/scripts/migrate_collection_comments.sh b/scripts/migrate_collection_comments.sh deleted file mode 100755 index b3f75422..00000000 --- a/scripts/migrate_collection_comments.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -# -# One-shot comments migration for a collection (run on the migration pod). -# -# Consolidates, in this order: -# 1. pre-flight checks (files from export_collection_comments.py are there, -# the collection is configured in streams.yaml) -# 2. invenio migration comments commenters-run (dry run, then real run) -# 3. invenio migration comments run (dry run, then real run) -# -# Usage: -# ./scripts/migrate_collection_comments.sh --collection antares_restr -# ./scripts/migrate_collection_comments.sh --collection antares_restr --yes -# ./scripts/migrate_collection_comments.sh --collection antares_restr --dry-run-only -# -# The input paths are NOT passed on the command line: the CLI reads them from the -# `comments:` section of streams.yaml (`dir_path`, plus `/users`). -# -# Commenters are indexed by the loader right when they are created, so no manual -# re-indexing is needed. Both runners can be re-run: already created users and -# comments are skipped. - -set -euo pipefail - -COLLECTION="" -ASSUME_YES=false -DRY_RUN_ONLY=false -STREAMS_FILE="${CDS_MIGRATOR_KIT_STREAM_CONFIG:-}" - -while [[ $# -gt 0 ]]; do - case "$1" in - --collection) COLLECTION="$2"; shift 2 ;; - --yes|-y) ASSUME_YES=true; shift ;; - --dry-run-only) DRY_RUN_ONLY=true; shift ;; - --streams-file) STREAMS_FILE="$2"; shift 2 ;; - -h|--help) sed -n '2,19p' "$0"; exit 0 ;; - *) echo "Unknown argument: $1" >&2; exit 2 ;; - esac -done - -[[ -n "$COLLECTION" ]] || { echo "--collection is required" >&2; exit 2; } - -# --- 1. pre-flight checks --------------------------------------------------- -# dir_path of the collection, from the same streams.yaml the runners use. -DIR_PATH="$(python - "$COLLECTION" "$STREAMS_FILE" <<'EOF' -import sys -import yaml -from flask import current_app -from invenio_app.factory import create_app - -collection, streams_file = sys.argv[1], sys.argv[2] -if not streams_file: - with create_app().app_context(): - streams_file = current_app.config["CDS_MIGRATOR_KIT_STREAM_CONFIG"] -with open(streams_file) as fp: - config = yaml.safe_load(fp) -try: - print(config["comments"][collection]["dir_path"]) -except KeyError: - sys.exit(f"collection '{collection}' has no entry under `comments:` in {streams_file}") -EOF -)" - -echo "Comments directory: $DIR_PATH" -missing=0 -for f in "comments_metadata.json" "users/missing_commentors_from_ldap.json"; do - if [[ ! -f "$DIR_PATH/$f" ]]; then - echo "MISSING: $DIR_PATH/$f (run scripts/export_collection_comments.py first)" >&2 - missing=1 - fi -done -[[ -f "$DIR_PATH/users/people.csv" ]] || echo "WARNING: $DIR_PATH/users/people.csv not found" -[[ $missing -eq 0 ]] || exit 1 - -confirm() { - $ASSUME_YES && return 0 - read -r -p "$1 [y/N] " answer - [[ "$answer" =~ ^[Yy]$ ]] -} - -# The runners catch and log their own errors, so a dry run exit code can't be -# trusted to tell success from failure: check the logs before confirming. -run_step() { - local name="$1" cmd="$2" - echo; echo "=== $name: dry run ===" - invenio migration comments "$cmd" --collection "$COLLECTION" --dry-run - $DRY_RUN_ONLY && return 0 - confirm "Dry run of '$name' finished. Check the logs, then run it for real?" \ - || { echo "Stopped before '$name'."; exit 1; } - echo "=== $name: real run ===" - invenio migration comments "$cmd" --collection "$COLLECTION" -} - -# --- 2. commenters, 3. comments -------------------------------------------- -run_step "commenters" commenters-run -run_step "comments" run - -echo; echo "Done. Re-run is safe: existing users and comments are skipped." diff --git a/tests/cds-rdm/helpers.py b/tests/cds-rdm/helpers.py index 6319a0bd..263974a2 100644 --- a/tests/cds-rdm/helpers.py +++ b/tests/cds-rdm/helpers.py @@ -39,7 +39,7 @@ def config(mocker, community, orcid_name_data): "communities_ids": [f"{str(community.id)}"], }, "load": { - "legacy_pids_to_redirect": "cds_migrator_kit/rdm/data/summer_student_reports/duplicated_pids.json" + "legacy_pids_to_redirect": "tests/cds-rdm/data/sspn/duplicated_pids.json" }, }, "it": { @@ -113,7 +113,7 @@ def config(mocker, community, orcid_name_data): "communities_ids": [f"{str(community.id)}"], }, "load": { - "legacy_pids_to_redirect": "cds_migrator_kit/rdm/data/summer_student_reports/duplicated_pids.json" + "legacy_pids_to_redirect": "tests/cds-rdm/data/sspn/duplicated_pids.json" }, }, "sspn_publication_date_consistency": { @@ -128,7 +128,7 @@ def config(mocker, community, orcid_name_data): "communities_ids": [f"{str(community.id)}"], }, "load": { - "legacy_pids_to_redirect": "cds_migrator_kit/rdm/data/summer_student_reports/duplicated_pids.json" + "legacy_pids_to_redirect": "tests/cds-rdm/data/sspn/duplicated_pids.json" }, }, "lcd_restr": {