diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e7a5cec..1f93b2b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2021 CERN. +# Copyright (C) 2021-2026 CERN. # -# Invenio App RDM is free software; you can redistribute it and/or modify it -# under the terms of the MIT License; see LICENSE file for more details. +# CDS-DoJSON is free software; you can redistribute it and/or modify it +# under the terms of the Revised BSD License; see LICENSE file for more details. name: CI @@ -21,18 +21,12 @@ on: jobs: Tests: - runs-on: ubuntu-20.04 - continue-on-error: ${{ matrix.experimental }} + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: [3.9] - requirements-level: [pypi] - experimental: [false] - include: - - python-version: 3.12 - experimental: true + python-version: ["3.9", "3.12", "3.14"] env: - DB: ${{ matrix.db-service }} EXTRAS: tests steps: @@ -50,8 +44,6 @@ jobs: run: | pip install ".[$EXTRAS]" pip freeze - docker --version - docker compose --version - name: Run tests run: ./run-tests.sh diff --git a/CHANGES.rst b/CHANGES.rst index ac7529e9..fea0f171 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changes ======= +Version 0.14.0 (released 2026-10-13) +----------------------------------- + +- chore(format): ran new black and isort formatters on code +- update(python): update python version support and tests Version 0.13.0 (released 2024-09-30) ----------------------------------- diff --git a/cds_dojson/__init__.py b/cds_dojson/__init__.py index 682c1537..c3acfbf4 100644 --- a/cds_dojson/__init__.py +++ b/cds_dojson/__init__.py @@ -19,4 +19,4 @@ """The CDS DoJson.""" -__version__ = "0.13.0" +__version__ = "0.14.0" diff --git a/cds_dojson/cli.py b/cds_dojson/cli.py index 57d51d77..3cdaab44 100644 --- a/cds_dojson/cli.py +++ b/cds_dojson/cli.py @@ -42,22 +42,22 @@ def cli(): @cli.command() -@click.argument('schema', type=click.Path(exists=True)) +@click.argument("schema", type=click.Path(exists=True)) def compile_schema(schema): """Compile ``$ref`` and ``allOf`` key from the given JSON schema.""" with open(schema) as f: schema_json = json.load(f) - base_uri = 'file://{0}/'.format(os.path.dirname(os.path.abspath(schema))) + base_uri = "file://{0}/".format(os.path.dirname(os.path.abspath(schema))) click.echo(json.dumps(_compile_schema(schema_json, base_uri), indent=2)) @cli.command() -@click.argument('source', type=click.Path(exists=True)) -@click.argument('destination', type=click.Path()) +@click.argument("source", type=click.Path(exists=True)) +@click.argument("destination", type=click.Path()) def convert_yaml2json(source, destination): """Convert files from yaml to json.""" if sys.version_info[0] <= 3 and sys.version_info[1] < 6: - click.echo('Use Python 3.6+ to deterministically generate the json.') + click.echo("Use Python 3.6+ to deterministically generate the json.") yaml2json(source, destination) diff --git a/cds_dojson/exceptions.py b/cds_dojson/exceptions.py index 374355a3..c0e181a7 100644 --- a/cds_dojson/exceptions.py +++ b/cds_dojson/exceptions.py @@ -16,4 +16,3 @@ class ModelMissingException(Exception): class MultipleModelsException(Exception): """CDSDoJSONException class.""" - diff --git a/cds_dojson/marc21/__init__.py b/cds_dojson/marc21/__init__.py index eb23309d..7062fbec 100644 --- a/cds_dojson/marc21/__init__.py +++ b/cds_dojson/marc21/__init__.py @@ -21,4 +21,4 @@ from ..overdo import OverdoBase -marc21 = OverdoBase(entry_point_models='cds_dojson.marc21.models') +marc21 = OverdoBase(entry_point_models="cds_dojson.marc21.models") diff --git a/cds_dojson/marc21/fields/base.py b/cds_dojson/marc21/fields/base.py index 45648a4c..435114c9 100644 --- a/cds_dojson/marc21/fields/base.py +++ b/cds_dojson/marc21/fields/base.py @@ -31,29 +31,27 @@ from ..models.base import model -@model.over('recid', '^001') +@model.over("recid", "^001") def recid(self, key, value): """Record Identifier.""" return int(value) -@model.over('agency_code', '^003') +@model.over("agency_code", "^003") def agency_code(self, key, value): """Control number identifier.""" - return 'SzGeCERN' + return "SzGeCERN" -@model.over('videos', '^774') +@model.over("videos", "^774") @for_each_value def videos(self, key, value): """Videos.""" - return { - '$ref': value.get('u') - } + return {"$ref": value.get("u")} -@model.over('modified_by', '^937__') +@model.over("modified_by", "^937__") @ignore_value def modified_by(self, key, value): """Modified by.""" - return value.get('s') + return value.get("s") diff --git a/cds_dojson/marc21/fields/books/base.py b/cds_dojson/marc21/fields/books/base.py index 943c840a..6cddeb92 100644 --- a/cds_dojson/marc21/fields/books/base.py +++ b/cds_dojson/marc21/fields/books/base.py @@ -60,195 +60,199 @@ from .utils import extract_parts, is_excluded -@model.over('legacy_recid', '^001') +@model.over("legacy_recid", "^001") def recid(self, key, value): """Record Identifier.""" return int(value) -@model.over('agency_code', '^003') +@model.over("agency_code", "^003") def agency_code(self, key, value): """Control number identifier.""" - return 'SzGeCERN' + return "SzGeCERN" -@model.over('created_by', '^859__') +@model.over("created_by", "^859__") @filter_values def created_by(self, key, value): """Translates created_by to find submitter.""" - return {'_email': clean_email(clean_val('f', value, str)), 'type': "user"} + return {"_email": clean_email(clean_val("f", value, str)), "type": "user"} -@model.over('_created', '(^916__)|(^595__)') +@model.over("_created", "(^916__)|(^595__)") @out_strip def created(self, key, value): """Translates created information to fields.""" - _created_by = self.get('created_by', {}) - - if key == '916__': - if 's' in value: - _created_by.update({'type': mapping(ACQUISITION_METHOD, - clean_val('s', value, str, - default="migration"), - raise_exception=True - )}) - self['created_by'] = _created_by - date = clean_val('w', value, int, regex_format=r'\d{4}$') + _created_by = self.get("created_by", {}) + + if key == "916__": + if "s" in value: + _created_by.update( + { + "type": mapping( + ACQUISITION_METHOD, + clean_val("s", value, str, default="migration"), + raise_exception=True, + ) + } + ) + self["created_by"] = _created_by + date = clean_val("w", value, int, regex_format=r"\d{4}$") if date: year, week = str(date)[:4], str(date)[4:] date = get_week_start(int(year), int(week)) return date.isoformat() - elif key == '595__': + elif key == "595__": try: - sub_a = clean_val('a', value, str, - regex_format=r'[A-Z]{3}[0-9]{6}$') + sub_a = clean_val("a", value, str, regex_format=r"[A-Z]{3}[0-9]{6}$") if sub_a: source = sub_a[:3] self["source"] = source year, month = int(sub_a[3:7]), int(sub_a[7:]) - self['_created'] = datetime.date(year, month, 1).isoformat() - raise IgnoreKey('_created') + self["_created"] = datetime.date(year, month, 1).isoformat() + raise IgnoreKey("_created") except UnexpectedValue as e: - e.subfield = 'a' - self['internal_notes'] = internal_notes(self, key, value) - raise IgnoreKey('_created') + e.subfield = "a" + self["internal_notes"] = internal_notes(self, key, value) + raise IgnoreKey("_created") - raise IgnoreKey('_created') + raise IgnoreKey("_created") -@model.over('internal_notes', '^595__') +@model.over("internal_notes", "^595__") def internal_notes(self, key, value): """Translates private notes field.""" - _internal_notes = self.get('internal_notes', []) + _internal_notes = self.get("internal_notes", []) for v in force_list(value): - internal_note = {'value': clean_val('a', v, str, req=True)} + internal_note = {"value": clean_val("a", v, str, req=True)} if internal_note not in _internal_notes: _internal_notes.append(internal_note) return _internal_notes # TODO convert to tags -@model.over('_migration', '(^980__)|(^690C_)|(^697C_)') +@model.over("_migration", "(^980__)|(^690C_)|(^697C_)") @out_strip def collection(self, key, value): """Translates collection field - WARNING - also document type field.""" - _migration = self['_migration'] - _tags = _migration['tags'] + _migration = self["_migration"] + _tags = _migration["tags"] for v in force_list(value): - result_a = mapping(COLLECTION, clean_val('a', v, str)) - result_b = mapping(COLLECTION, clean_val('b', v, str)) + result_a = mapping(COLLECTION, clean_val("a", v, str)) + result_b = mapping(COLLECTION, clean_val("b", v, str)) if result_a: _tags.append(result_a) if result_a not in _tags else None - _migration['has_tags'] = True + _migration["has_tags"] = True if result_b: _tags.append(result_b) if result_b not in _tags else None - _migration['has_tags'] = True + _migration["has_tags"] = True if not result_a and not result_b: - self['document_type'] = document_type(self, key, value) - raise IgnoreKey('_migration') + self["document_type"] = document_type(self, key, value) + raise IgnoreKey("_migration") return _migration -@model.over('document_type', '(^980__)|(^960__)|(^690C_)') +@model.over("document_type", "(^980__)|(^960__)|(^690C_)") @out_strip def document_type(self, key, value): """Translates document type field.""" - _doc_type = self.get('document_type', {}) + _doc_type = self.get("document_type", {}) def doc_type_mapping(val): if val: return mapping(DOCUMENT_TYPE, val) for v in force_list(value): - val_a = doc_type_mapping(clean_val('a', v, str)) - val_b = doc_type_mapping(clean_val('b', v, str)) + val_a = doc_type_mapping(clean_val("a", v, str)) + val_b = doc_type_mapping(clean_val("b", v, str)) if not val_a and not val_b and not _doc_type: - raise UnexpectedValue(subfield='a') + raise UnexpectedValue(subfield="a") if val_a and val_b and (val_a != val_b != _doc_type): - raise ManualMigrationRequired(subfield='a or b - ' - 'inconsistent doc type') + raise ManualMigrationRequired(subfield="a or b - " "inconsistent doc type") if val_a: if _doc_type and _doc_type != val_a: - raise ManualMigrationRequired(subfield='a' - 'inconsistent doc type') + raise ManualMigrationRequired(subfield="a" "inconsistent doc type") _doc_type = val_a if val_b: if _doc_type and _doc_type != val_a: - raise ManualMigrationRequired(subfield='b' - 'inconsistent doc type') + raise ManualMigrationRequired(subfield="b" "inconsistent doc type") _doc_type = val_b return _doc_type -@model.over('authors', '(^100__)|(^700__)') +@model.over("authors", "(^100__)|(^700__)") @filter_list_values def authors(self, key, value): """Translates the authors field.""" - _authors = self.get('authors', []) + _authors = self.get("authors", []) item = build_contributor_books(value) if item and item not in _authors: _authors.append(item) try: - if 'u' in value: - other = ['et al.', 'et al'] - val_u = list(force_list(value.get('u'))) + if "u" in value: + other = ["et al.", "et al"] + val_u = list(force_list(value.get("u"))) if [i for i in other if i in val_u]: - self['other_authors'] = True + self["other_authors"] = True except UnexpectedValue: pass return _authors -@model.over('authors', '^720__') +@model.over("authors", "^720__") @filter_list_values def alt_authors(self, key, value): """Translates the alternative authors field.""" - _authors = self.get('authors', []) + _authors = self.get("authors", []) if _authors: for i, v in enumerate(force_list(value)): - _authors[i].update({'alternative_names': clean_val('a', v, str)}) + _authors[i].update({"alternative_names": clean_val("a", v, str)}) return _authors -@model.over('authors', '(^110)|(^710_[a_]+)') +@model.over("authors", "(^110)|(^710_[a_]+)") @filter_list_values def corporate_authors(self, key, value): """Translates the corporate authors field.""" - _corporate_authors = self.get('authors', []) + _corporate_authors = self.get("authors", []) for v in force_list(value): - if key == '710__': - if 'a' in v: - _corporate_authors.append({'full_name': clean_val('a', v, str), - 'type': 'ORGANISATION'}) + if key == "710__": + if "a" in v: + _corporate_authors.append( + {"full_name": clean_val("a", v, str), "type": "ORGANISATION"} + ) else: - self['authors'] = collaborations(self, key, value) - raise IgnoreKey('corporate_authors') + self["authors"] = collaborations(self, key, value) + raise IgnoreKey("corporate_authors") else: - _corporate_authors.append({'full_name': clean_val('a', v, str), - 'type': 'ORGANISATION'}) + _corporate_authors.append( + {"full_name": clean_val("a", v, str), "type": "ORGANISATION"} + ) return _corporate_authors -@model.over('authors', '^710__') -@replace_in_result('Collaboration', '', key='full_name') +@model.over("authors", "^710__") +@replace_in_result("Collaboration", "", key="full_name") @filter_list_values def collaborations(self, key, value): """Translates collaborations.""" - _authors = self.get('authors', []) + _authors = self.get("authors", []) for v in force_list(value): - if 'g' in v: - _authors.append({'full_name': clean_val('g', v, str), - 'type': 'ORGANISATION'}) - elif '5' in v: - _authors.append({'full_name': clean_val('5', v, str), - 'type': 'ORGANISATION'}) + if "g" in v: + _authors.append( + {"full_name": clean_val("g", v, str), "type": "ORGANISATION"} + ) + elif "5" in v: + _authors.append( + {"full_name": clean_val("5", v, str), "type": "ORGANISATION"} + ) return _authors -@model.over('publication_info', '(^773__)') +@model.over("publication_info", "(^773__)") @filter_list_values def publication_info(self, key, value): """Translates publication_info field. @@ -256,67 +260,71 @@ def publication_info(self, key, value): if x and o subfields are present simultaneously it concatenates the text """ - _publication_info = self.get('publication_info', []) + _publication_info = self.get("publication_info", []) for v in force_list(value): temp_info = {} - pages = clean_pages_range('c', v) + pages = clean_pages_range("c", v) if pages: temp_info.update(pages) - temp_info.update({ - 'journal_issue': clean_val('n', v, str), - 'journal_title': clean_val('p', v, str), - 'journal_volume': clean_val('v', v, str), - 'year': clean_val('y', v, int), - }) - - text = '{0} {1}'.format( - clean_val('o', v, str) or '', - clean_val('x', v, str) or '').strip() + temp_info.update( + { + "journal_issue": clean_val("n", v, str), + "journal_title": clean_val("p", v, str), + "journal_volume": clean_val("v", v, str), + "year": clean_val("y", v, int), + } + ) + + text = "{0} {1}".format( + clean_val("o", v, str) or "", clean_val("x", v, str) or "" + ).strip() if text: - temp_info.update({'note': text}) + temp_info.update({"note": text}) if temp_info: _publication_info.append(temp_info) return _publication_info -@model.over('extensions', '(^925__)') +@model.over("extensions", "(^925__)") @filter_values def standard_review(self, key, value): """Translates standard_status field.""" - _extensions = self.get('extensions', {}) + _extensions = self.get("extensions", {}) _extensions.update( - {'standard_review:applicability': clean_val('i', value, str), - 'standard_review:validity': clean_val('v', value, str), - 'standard_review:checkdate': clean_val('z', value, str), - 'standard_review:expert': clean_val('p', value, str) - }) + { + "standard_review:applicability": clean_val("i", value, str), + "standard_review:validity": clean_val("v", value, str), + "standard_review:checkdate": clean_val("z", value, str), + "standard_review:expert": clean_val("p", value, str), + } + ) return _extensions -@model.over('publication_info', '^962__') +@model.over("publication_info", "^962__") def publication_additional(self, key, value): """Translates additional publication info.""" - _publication_info = self.get('publication_info', []) + _publication_info = self.get("publication_info", []) _migration = self["_migration"] empty = not bool(_publication_info) for i, v in enumerate(force_list(value)): temp_info = {} - pages = clean_pages_range('k', v) + pages = clean_pages_range("k", v) if pages: temp_info.update(pages) - rel_recid = clean_val('b', v, str) + rel_recid = clean_val("b", v, str) if rel_recid: _migration["journal_record_legacy_recid"] = rel_recid _migration["has_journal"] = True # assume that if we have a parent journal # then the doc is a periodical issue - self['document_type'] = 'PERIODICAL_ISSUE' - n_subfield = clean_val('n', v, str) - if n_subfield.upper() == 'BOOK': - temp_info.update({'material': 'BOOK'}) + self["document_type"] = "PERIODICAL_ISSUE" + n_subfield = clean_val("n", v, str) + if n_subfield.upper() == "BOOK": + temp_info.update({"material": "BOOK"}) else: - _conference_info = self.get('conference_info', {}) - _identifiers = _conference_info.get('identifiers', []) + _conference_info = self.get("conference_info", {}) + _identifiers = _conference_info.get("identifiers", []) conf_id = {"scheme": "CERN_CODE", "value": n_subfield} _identifiers.append(conf_id) _conference_info["identifiers"] = _identifiers @@ -330,47 +338,50 @@ def publication_additional(self, key, value): # cleaning data, later -@model.over('_migration', '(^775__)|(^787__)') +@model.over("_migration", "(^775__)|(^787__)") @out_strip def related_records(self, key, value): """Translates related_records field. RELATED records """ - _migration = self['_migration'] - _related = _migration['related'] - relation_type = 'other' + _migration = self["_migration"] + _related = _migration["related"] + relation_type = "other" try: - if key == '775__' and 'b' in value: - relation_type = clean_val('b', value, str) - if key == '787__' and 'i' in value: - clean_val('i', value, str, manual=True) - _related.append({'related_recid': clean_val('w', value, str, req=True), - 'relation_type': relation_type - }) - _migration.update({'related': _related, 'has_related': True}) + if key == "775__" and "b" in value: + relation_type = clean_val("b", value, str) + if key == "787__" and "i" in value: + clean_val("i", value, str, manual=True) + _related.append( + { + "related_recid": clean_val("w", value, str, req=True), + "relation_type": relation_type, + } + ) + _migration.update({"related": _related, "has_related": True}) return _migration except ManualMigrationRequired as e: - if key == '775__': - e.subfield = 'b or c' + if key == "775__": + e.subfield = "b or c" else: - e.subfield = 'i' + e.subfield = "i" raise e -@model.over('extensions', '^693__') +@model.over("extensions", "^693__") @filter_values def accelerator_experiments(self, key, value): """Translates accelerator_experiments field.""" - _extensions = self.get('extensions', {}) + _extensions = self.get("extensions", {}) - sub_a = clean_val('a', value, str) - sub_e = clean_val('e', value, str) - sub_p = clean_val('p', value, str) + sub_a = clean_val("a", value, str) + sub_e = clean_val("e", value, str) + sub_p = clean_val("p", value, str) - accelerators = _extensions.get('unit:accelerator', []) - experiment = _extensions.get('unit:experiment', []) - project = _extensions.get('unit:project', []) + accelerators = _extensions.get("unit:accelerator", []) + experiment = _extensions.get("unit:experiment", []) + project = _extensions.get("unit:project", []) if sub_a and sub_a not in accelerators: accelerators.append(sub_a) @@ -379,234 +390,234 @@ def accelerator_experiments(self, key, value): if sub_p and sub_p not in project: project.append(sub_p) - _extensions.update({ - "unit:accelerator": accelerators, - "unit:experiment": experiment, - "unit:project": project - }) + _extensions.update( + { + "unit:accelerator": accelerators, + "unit:experiment": experiment, + "unit:project": project, + } + ) return _extensions -@model.over('urls', '^8564_') +@model.over("urls", "^8564_") @filter_list_values @for_each_value def urls(self, key, value): """Translates urls field.""" - sub_y = clean_val('y', value, str, default='') - sub_u = clean_val('u', value, str, req=True) + sub_y = clean_val("y", value, str, default="") + sub_u = clean_val("u", value, str, req=True) - eitems_ebl = self['_migration']['eitems_ebl'] - eitems_external = self['_migration']['eitems_external'] - eitems_proxy = self['_migration']['eitems_proxy'] - eitems_files = self['_migration']['eitems_file_links'] + eitems_ebl = self["_migration"]["eitems_ebl"] + eitems_external = self["_migration"]["eitems_external"] + eitems_proxy = self["_migration"]["eitems_proxy"] + eitems_files = self["_migration"]["eitems_file_links"] - url = {'value': sub_u} - if sub_y and sub_y != 'ebook': - url['description'] = sub_y + url = {"value": sub_u} + if sub_y and sub_y != "ebook": + url["description"] = sub_y # EBL publisher login required - if all([elem in sub_u for elem in ['cds', '.cern.ch' '/auth.py']]): + if all([elem in sub_u for elem in ["cds", ".cern.ch" "/auth.py"]]): eitems_ebl.append(url) - self['_migration']['eitems_has_ebl'] = True + self["_migration"]["eitems_has_ebl"] = True # EzProxy links - elif 'ezproxy.cern.ch' in sub_u: - url['value'] = url['value'].replace( - 'https://ezproxy.cern.ch/login?url=', '') + elif "ezproxy.cern.ch" in sub_u: + url["value"] = url["value"].replace("https://ezproxy.cern.ch/login?url=", "") eitems_proxy.append(url) - self['_migration']['eitems_has_proxy'] = True + self["_migration"]["eitems_has_proxy"] = True # local files - elif all([elem in sub_u for elem in - ['cds', '.cern.ch/record/', '/files']]): + elif all([elem in sub_u for elem in ["cds", ".cern.ch/record/", "/files"]]): eitems_files.append(url) - self['_migration']['eitems_has_files'] = True - elif sub_y == 'ebook' or sub_y == 'e-proceedings': + self["_migration"]["eitems_has_files"] = True + elif sub_y == "ebook" or sub_y == "e-proceedings": eitems_external.append(url) - self['_migration']['eitems_has_external'] = True + self["_migration"]["eitems_has_external"] = True else: # if none of the above, it is just external url # attached to the document return url -@model.over('identifiers', '^020__', ) +@model.over( + "identifiers", + "^020__", +) @filter_list_values def isbns(self, key, value): """Translates isbns fields.""" - _isbns = self.get('identifiers', []) + _isbns = self.get("identifiers", []) for v in force_list(value): - subfield_u = clean_val('u', v, str) - isbn = {'value': clean_val('a', v, str) or clean_val('z', v, str), - 'scheme': 'ISBN', - } - if not isbn['value']: - raise ManualMigrationRequired(subfield='a or z') + subfield_u = clean_val("u", v, str) + isbn = { + "value": clean_val("a", v, str) or clean_val("z", v, str), + "scheme": "ISBN", + } + if not isbn["value"]: + raise ManualMigrationRequired(subfield="a or z") if subfield_u: - volume = re.search(r'(\(*v[.| ]*\d+.*\)*)', subfield_u) + volume = re.search(r"(\(*v[.| ]*\d+.*\)*)", subfield_u) if volume: volume = volume.group(1) - subfield_u = subfield_u.replace(volume, '').strip() - existing_volume = self.get('volume') + subfield_u = subfield_u.replace(volume, "").strip() + existing_volume = self.get("volume") if existing_volume: - raise ManualMigrationRequired(subfield='u') + raise ManualMigrationRequired(subfield="u") # TODO volume --> when splitting to series - self['volume'] = volume + self["volume"] = volume if subfield_u.upper() in MEDIUM_TYPES: - isbn.update({'medium': subfield_u}) + isbn.update({"medium": subfield_u}) else: - isbn.update({'description': subfield_u}) + isbn.update({"description": subfield_u}) # TODO subfield C if isbn not in _isbns: _isbns.append(isbn) return _isbns -@model.over('identifiers', '^021__') +@model.over("identifiers", "^021__") @filter_list_values def standard_numbers(self, key, value): """Translates standard numbers values.""" - _identifiers = self.get('identifiers', []) - a = clean_val('a', value, str) - b = clean_val('b', value, str) + _identifiers = self.get("identifiers", []) + a = clean_val("a", value, str) + b = clean_val("b", value, str) sn = a or b if sn: - _identifiers.append({'value': sn, - 'scheme': 'STANDARD_NUMBER', - 'hidden': True if b else False}) + _identifiers.append( + {"value": sn, "scheme": "STANDARD_NUMBER", "hidden": True if b else False} + ) return _identifiers - raise MissingRequiredField(subfield='a or b') + raise MissingRequiredField(subfield="a or b") -@model.over('alternative_identifiers', '(^0247_)|(^035__)|(^036__)') +@model.over("alternative_identifiers", "(^0247_)|(^035__)|(^036__)") @for_each_value @filter_values def alternative_identifiers(self, key, value): """Translates external_system_identifiers fields.""" - field_type = clean_val('2', value, str) - sub_a = clean_val('a', value, str, req=True) + field_type = clean_val("2", value, str) + sub_a = clean_val("a", value, str, req=True) indentifier_entry = {} - if key == '0247_': - if field_type and field_type.lower() == 'doi': + if key == "0247_": + if field_type and field_type.lower() == "doi": # if 0247__2 == doi it is a DOI identifier - self['identifiers'] = dois(self, key, value) - raise IgnoreKey('alternative_identifiers') - elif field_type and field_type.upper() in \ - EXTERNAL_SYSTEM_IDENTIFIERS_TO_IGNORE: - raise IgnoreKey('alternative_identifiers') + self["identifiers"] = dois(self, key, value) + raise IgnoreKey("alternative_identifiers") + elif field_type and field_type.upper() in EXTERNAL_SYSTEM_IDENTIFIERS_TO_IGNORE: + raise IgnoreKey("alternative_identifiers") else: - raise UnexpectedValue(subfield='2') - if key == '035__': - if 'CERCER' in sub_a: - raise IgnoreKey('alternative_identifiers') - sub_9 = clean_val('9', value, str, req=True) - if 'CERCER' in sub_9: - raise IgnoreKey('alternative_identifiers') + raise UnexpectedValue(subfield="2") + if key == "035__": + if "CERCER" in sub_a: + raise IgnoreKey("alternative_identifiers") + sub_9 = clean_val("9", value, str, req=True) + if "CERCER" in sub_9: + raise IgnoreKey("alternative_identifiers") # conference_info.identifiers mixed data - if sub_9.upper() == 'INSPIRE-CNUM': - _conference_info = self.get('conference_info', {}) - _prev_identifiers = _conference_info.get('identifiers', []) - _prev_identifiers.append( - {'scheme': 'INSPIRE_CNUM', 'value': sub_a}) - _conference_info.update({'identifiers': _prev_identifiers}) - self['conference_info'] = _conference_info - raise IgnoreKey('alternative_identifiers') + if sub_9.upper() == "INSPIRE-CNUM": + _conference_info = self.get("conference_info", {}) + _prev_identifiers = _conference_info.get("identifiers", []) + _prev_identifiers.append({"scheme": "INSPIRE_CNUM", "value": sub_a}) + _conference_info.update({"identifiers": _prev_identifiers}) + self["conference_info"] = _conference_info + raise IgnoreKey("alternative_identifiers") elif sub_9.upper() in EXTERNAL_SYSTEM_IDENTIFIERS: - indentifier_entry.update({'value': sub_a, - 'scheme': sub_9}) + indentifier_entry.update({"value": sub_a, "scheme": sub_9}) elif sub_9.upper() in EXTERNAL_SYSTEM_IDENTIFIERS_TO_IGNORE: - raise IgnoreKey('external_system_identifiers') + raise IgnoreKey("external_system_identifiers") else: - raise UnexpectedValue(subfield='9') - if key == '036__': - indentifier_entry.update({ - 'value': sub_a, 'scheme': clean_val('9', value, str, req=True)}) + raise UnexpectedValue(subfield="9") + if key == "036__": + indentifier_entry.update( + {"value": sub_a, "scheme": clean_val("9", value, str, req=True)} + ) return indentifier_entry -@model.over('identifiers', '^0247_') +@model.over("identifiers", "^0247_") @filter_list_values def dois(self, key, value): """Translates dois fields.""" - _identifiers = self.get('identifiers', []) + _identifiers = self.get("identifiers", []) for v in force_list(value): - material = mapping(MATERIALS, - clean_val('q', v, str, transform='lower'), - raise_exception=True) + material = mapping( + MATERIALS, clean_val("q", v, str, transform="lower"), raise_exception=True + ) doi = { - 'value': clean_val('a', v, str, req=True), - 'material': material, - 'source': clean_val('9', v, str), - 'scheme': 'DOI', + "value": clean_val("a", v, str, req=True), + "material": material, + "source": clean_val("9", v, str), + "scheme": "DOI", } if doi not in _identifiers: _identifiers.append(doi) return _identifiers -@model.over('identifiers', '(^037__)|(^088__)') +@model.over("identifiers", "(^037__)|(^088__)") @filter_list_values def report_numbers(self, key, value): """Translates report_numbers fields.""" def get_value_rn(f_a, f_z, f_9, rn_obj): - rn_obj.update({'value': f_a or f_z or f_9, 'scheme': 'REPORT_NUMBER'}) + rn_obj.update({"value": f_a or f_z or f_9, "scheme": "REPORT_NUMBER"}) if f_z or f_9: - rn_obj.update({'hidden': True}) + rn_obj.update({"hidden": True}) - _identifiers = self.get('identifiers', []) + _identifiers = self.get("identifiers", []) - sub_9 = clean_val('9', value, str) - sub_a = clean_val('a', value, str) - sub_z = clean_val('z', value, str) + sub_9 = clean_val("9", value, str) + sub_a = clean_val("a", value, str) + sub_z = clean_val("z", value, str) all_empty = not (sub_z or sub_a or sub_9) - if key == '037__': + if key == "037__": entry = {} if all_empty: - raise MissingRequiredField(subfield='9 or a or z') + raise MissingRequiredField(subfield="9 or a or z") - if sub_9 == 'arXiv': + if sub_9 == "arXiv": arxiv_eprints(self, key, value) - raise IgnoreKey('identifiers') + raise IgnoreKey("identifiers") else: get_value_rn(sub_a, sub_z, sub_9, entry) _identifiers.append(entry) - if key == '088__': + if key == "088__": entry = {} - if 'n' in value or 'x' in value: + if "n" in value or "x" in value: barcodes(self, key, value) - if all_empty and 'n' not in value and 'x' not in value: - raise MissingRequiredField(subfield='9 or a or z or n or x') + if all_empty and "n" not in value and "x" not in value: + raise MissingRequiredField(subfield="9 or a or z or n or x") get_value_rn(sub_a, sub_z, sub_9, entry) _identifiers.append(entry) return _identifiers -@model.over('barcodes', '^088__') +@model.over("barcodes", "^088__") @for_each_value def barcodes(self, key, value): """Match barcodes of items to volumes.""" - val_n = clean_val('n', value, str) - val_x = clean_val('x', value, str) - - _migration = self['_migration'] - _migration['volumes'].append(dict( - volume=extract_volume_number( - val_n, - raise_exception=True, - subfield='n' - ), - barcode=val_x - )) - raise IgnoreKey('barcodes') - - -@model.over('subjects', '(^037__)') + val_n = clean_val("n", value, str) + val_x = clean_val("x", value, str) + + _migration = self["_migration"] + _migration["volumes"].append( + dict( + volume=extract_volume_number(val_n, raise_exception=True, subfield="n"), + barcode=val_x, + ) + ) + raise IgnoreKey("barcodes") + + +@model.over("subjects", "(^037__)") @filter_list_values def arxiv_eprints(self, key, value): """Translates arxiv_eprints fields. @@ -624,267 +635,275 @@ def check_category(field, val): return category raise UnexpectedValue(subfield=field) - if key == '037__': - _alternative_identifiers = self.get('alternative_identifiers', []) + if key == "037__": + _alternative_identifiers = self.get("alternative_identifiers", []) for v in force_list(value): - eprint_id = clean_val('a', v, str, req=True) + eprint_id = clean_val("a", v, str, req=True) duplicated = [ elem for i, elem in enumerate(_alternative_identifiers) - if elem['value'] == eprint_id and elem['scheme'] == 'arXiv' + if elem["value"] == eprint_id and elem["scheme"] == "arXiv" ] - category = check_category('c', v) + category = check_category("c", v) if not duplicated: - eprint = {'value': eprint_id, 'scheme': 'arXiv'} + eprint = {"value": eprint_id, "scheme": "arXiv"} _alternative_identifiers.append(eprint) - self['alternative_identifiers'] = _alternative_identifiers + self["alternative_identifiers"] = _alternative_identifiers if category: - _subjects = self.get('subjects', []) - subject = {'scheme': 'arXiv', 'value': category} + _subjects = self.get("subjects", []) + subject = {"scheme": "arXiv", "value": category} _subjects.append(subject) if subject not in _subjects else None - self['subjects'] = _subjects - raise IgnoreKey('subjects') + self["subjects"] = _subjects + raise IgnoreKey("subjects") -@model.over('languages', '^041__') +@model.over("languages", "^041__") @for_each_value @out_strip def languages(self, key, value): """Translates languages fields.""" - lang = clean_val('a', value, str).lower() + lang = clean_val("a", value, str).lower() try: return pycountry.languages.lookup(lang).alpha_2.upper() except (KeyError, AttributeError, LookupError): - raise UnexpectedValue(subfield='a') + raise UnexpectedValue(subfield="a") -@model.over('subjects', - '(^050)|(^080__)|(^08204)|(^084__)|(^082__)') +@model.over("subjects", "(^050)|(^080__)|(^08204)|(^084__)|(^082__)") @for_each_value @out_strip def subject_classification(self, key, value): """Translates subject classification field.""" - prev_subjects = self.get('subjects', []) - _subject_classification = {'value': clean_val('a', value, str, req=True)} - if key == '080__': - _subject_classification.update({'scheme': 'UDC'}) - elif key.startswith('082'): - _subject_classification.update({'scheme': 'Dewey'}) - elif key == '084__': - sub_2 = clean_val('2', value, str) + prev_subjects = self.get("subjects", []) + _subject_classification = {"value": clean_val("a", value, str, req=True)} + if key == "080__": + _subject_classification.update({"scheme": "UDC"}) + elif key.startswith("082"): + _subject_classification.update({"scheme": "Dewey"}) + elif key == "084__": + sub_2 = clean_val("2", value, str) if sub_2 and sub_2.upper() in SUBJECT_CLASSIFICATION_EXCEPTIONS: keywords(self, key, value) - raise IgnoreKey('subjects') + raise IgnoreKey("subjects") else: - _subject_classification.update({'scheme': 'ICS'}) - elif key.startswith('050'): - _subject_classification.update({'scheme': 'LoC'}) + _subject_classification.update({"scheme": "ICS"}) + elif key.startswith("050"): + _subject_classification.update({"scheme": "LoC"}) if _subject_classification not in prev_subjects: return _subject_classification else: - raise IgnoreKey('subjects') + raise IgnoreKey("subjects") -@model.over('keywords', '^6531_') +@model.over("keywords", "^6531_") @for_each_value @filter_values def keywords(self, key, value): """Keywords.""" return { - 'value': clean_val('a', value, str), - 'source': value.get('9') or value.get('g'), + "value": clean_val("a", value, str), + "source": value.get("9") or value.get("g"), } -@model.over('conference_info', '(^111__)|(^711__)') +@model.over("conference_info", "(^111__)|(^711__)") @filter_values def conference_info(self, key, value): """Translates conference info.""" - _conference_info = self.get('conference_info', {}) + _conference_info = self.get("conference_info", {}) for v in force_list(value): - if key == '111__': + if key == "111__": try: - opening_date = parser.parse(clean_val('9', v, str, req=True)) - closing_date = parser.parse(clean_val('z', v, str, req=True)) - dates = "{0} - {1}".format(opening_date.date().isoformat(), - closing_date.date().isoformat()) + opening_date = parser.parse(clean_val("9", v, str, req=True)) + closing_date = parser.parse(clean_val("z", v, str, req=True)) + dates = "{0} - {1}".format( + opening_date.date().isoformat(), closing_date.date().isoformat() + ) except ValueError: - raise UnexpectedValue(subfield='9 or z') - country_code = clean_val('w', v, str) + raise UnexpectedValue(subfield="9 or z") + country_code = clean_val("w", v, str) if country_code: try: - country_code = str(pycountry.countries.get( - alpha_2=country_code).alpha_2) + country_code = str( + pycountry.countries.get(alpha_2=country_code).alpha_2 + ) except (KeyError, AttributeError): - raise UnexpectedValue(subfield='w') + raise UnexpectedValue(subfield="w") try: - series_number = clean_val('n', v, int) + series_number = clean_val("n", v, int) except TypeError: - raise UnexpectedValue('n', message=' series number not an int') + raise UnexpectedValue("n", message=" series number not an int") - _prev_identifiers = _conference_info.get('identifiers', []) + _prev_identifiers = _conference_info.get("identifiers", []) _prev_identifiers.append( - {'scheme': 'CERN_CODE', 'value': clean_val('g', v, str)}) - - _conference_info.update({ - 'title': clean_val('a', v, str, req=True), - 'place': clean_val('c', v, str, req=True), - 'dates': dates, - 'identifiers': _prev_identifiers, - 'series': {'number': series_number}, - 'country': country_code, - }) + {"scheme": "CERN_CODE", "value": clean_val("g", v, str)} + ) + + _conference_info.update( + { + "title": clean_val("a", v, str, req=True), + "place": clean_val("c", v, str, req=True), + "dates": dates, + "identifiers": _prev_identifiers, + "series": {"number": series_number}, + "country": country_code, + } + ) else: - _conference_info.update({ - 'acronym': clean_val('a', v, str)}) + _conference_info.update({"acronym": clean_val("a", v, str)}) return _conference_info -@model.over('alternative_titles', '^242__') +@model.over("alternative_titles", "^242__") @filter_list_values def alternative_titles(self, key, value): """Translates title translations.""" - _alternative_titles = self.get('alternative_titles', []) - if 'a' in value: - _alternative_titles.append({ - 'value': clean_val('a', value, str, req=True), - 'type': 'TRANSLATED_TITLE', - 'language': 'EN', - }) - if 'b' in value: - _alternative_titles.append({ - 'value': clean_val('b', value, str, req=True), - 'type': 'TRANSLATED_SUBTITLE', - 'language': 'EN', - }) + _alternative_titles = self.get("alternative_titles", []) + if "a" in value: + _alternative_titles.append( + { + "value": clean_val("a", value, str, req=True), + "type": "TRANSLATED_TITLE", + "language": "EN", + } + ) + if "b" in value: + _alternative_titles.append( + { + "value": clean_val("b", value, str, req=True), + "type": "TRANSLATED_SUBTITLE", + "language": "EN", + } + ) return _alternative_titles -@model.over('edition', '^250__') +@model.over("edition", "^250__") @out_strip def edition(self, key, value): """Translates edition indicator field.""" - return clean_val('a', value, str).replace("ed.", "") + return clean_val("a", value, str).replace("ed.", "") -@model.over('imprint', '^260__') +@model.over("imprint", "^260__") @filter_values def imprint(self, key, value): """Translates imprints fields.""" - reprint = clean_val('g', value, str) + reprint = clean_val("g", value, str) if reprint: - reprint = reprint.lower().replace('repr.', '').strip() + reprint = reprint.lower().replace("repr.", "").strip() try: - date = parser.parse(clean_val('c', value, str, req=True)) + date = parser.parse(clean_val("c", value, str, req=True)) except ParserError: - raise UnexpectedValue(subfield='c') + raise UnexpectedValue(subfield="c") except Exception: - raise UnexpectedValue(subfield='c') - self['publication_year'] = str(date.date().year) + raise UnexpectedValue(subfield="c") + self["publication_year"] = str(date.date().year) return { - 'date': clean_val('c', value, str, req=True), - 'place': clean_val('a', value, str), - 'publisher': clean_val('b', value, str), - 'reprint_date': reprint, + "date": clean_val("c", value, str, req=True), + "place": clean_val("a", value, str), + "publisher": clean_val("b", value, str), + "reprint_date": reprint, } -@model.over('book_series', '^490__') +@model.over("book_series", "^490__") @for_each_value def book_series(self, key, value): """Match barcodes to volumes.""" - val_n = clean_val('n', value, str) - val_x = clean_val('x', value, str) + val_n = clean_val("n", value, str) + val_x = clean_val("x", value, str) - _migration = self['_migration'] - _migration['serials'].append({ - 'title': clean_val('a', value, str), - 'volume': clean_val('v', value, str), - 'issn': val_x - }) - _migration['has_serial'] = True - raise IgnoreKey('book_series') + _migration = self["_migration"] + _migration["serials"].append( + { + "title": clean_val("a", value, str), + "volume": clean_val("v", value, str), + "issn": val_x, + } + ) + _migration["has_serial"] = True + raise IgnoreKey("book_series") -@model.over('note', '^500__') +@model.over("note", "^500__") @out_strip def note(self, key, value): """Translates public notes.""" # merge all found notes - _note = self.get('note', "") + _note = self.get("note", "") if _note: - _note = \ - "{0} / {1}".format(_note, clean_val('a', value, str, req=True)) + _note = "{0} / {1}".format(_note, clean_val("a", value, str, req=True)) else: - _note = clean_val('a', value, str, req=True) + _note = clean_val("a", value, str, req=True) return _note -@model.over('alternative_abstracts', '^520__') +@model.over("alternative_abstracts", "^520__") @for_each_value @out_strip def alternative_abstracts(self, key, value): """Translates abstracts fields.""" - abstract = self.get('abstract', None) - _alternative_abstracts = self.get('alternative_abstracts', []) + abstract = self.get("abstract", None) + _alternative_abstracts = self.get("alternative_abstracts", []) if not abstract: # takes first abstract as main - self["abstract"] = clean_val('a', value, str, req=True) - raise IgnoreKey('alternative_abstracts') - new_abstract = clean_val('a', value, str, req=True) + self["abstract"] = clean_val("a", value, str, req=True) + raise IgnoreKey("alternative_abstracts") + new_abstract = clean_val("a", value, str, req=True) return new_abstract if new_abstract not in _alternative_abstracts else None -@model.over('licenses', '^540__') +@model.over("licenses", "^540__") @for_each_value @filter_values def licenses(self, key, value): """Translates license fields.""" - material = mapping(MATERIALS, - clean_val('3', value, str, transform='lower'), - raise_exception=True) + material = mapping( + MATERIALS, clean_val("3", value, str, transform="lower"), raise_exception=True + ) return { - 'license': { - 'url': clean_val('u', value, str), - 'name': clean_val('a', value, str), + "license": { + "url": clean_val("u", value, str), + "name": clean_val("a", value, str), }, - 'material': material, - 'internal_note': clean_val('g', value, str), + "material": material, + "internal_note": clean_val("g", value, str), } -@model.over('copyrights', '^542__') +@model.over("copyrights", "^542__") @for_each_value @filter_values def copyright(self, key, value): """Translates copyright fields.""" - material = mapping(MATERIALS, - clean_val('3', value, str, transform='lower'), - raise_exception=True) + material = mapping( + MATERIALS, clean_val("3", value, str, transform="lower"), raise_exception=True + ) return { - 'material': material, - 'holder': clean_val('d', value, str), - 'statement': clean_val('f', value, str), - 'year': clean_val('g', value, int), - 'url': clean_val('u', value, str), + "material": material, + "holder": clean_val("d", value, str), + "statement": clean_val("f", value, str), + "year": clean_val("g", value, int), + "url": clean_val("u", value, str), } -@model.over('table_of_content', '(^505__)|(^5050_)') +@model.over("table_of_content", "(^505__)|(^5050_)") @out_strip @flatten @for_each_value def table_of_content(self, key, value): """Translates table of content field.""" - text = '{0} -- {1}'.format( - clean_val('a', value, str) or '', - clean_val('t', value, str) or '').strip() - if text != '--': - chapters = re.split(r'; | -- |--', text) + text = "{0} -- {1}".format( + clean_val("a", value, str) or "", clean_val("t", value, str) or "" + ).strip() + if text != "--": + chapters = re.split(r"; | -- |--", text) return chapters else: - raise UnexpectedValue(subfield='a or t') + raise UnexpectedValue(subfield="a or t") diff --git a/cds_dojson/marc21/fields/books/book.py b/cds_dojson/marc21/fields/books/book.py index a16d5d02..5e1669c2 100644 --- a/cds_dojson/marc21/fields/books/book.py +++ b/cds_dojson/marc21/fields/books/book.py @@ -39,74 +39,80 @@ from .base import alternative_titles as alternative_titles_base -@model.over('alternative_titles', '(^246__)|(^242__)') +@model.over("alternative_titles", "(^246__)|(^242__)") @filter_list_values def alternative_titles(self, key, value): """Alternative titles.""" - _alternative_titles = self.get('alternative_titles', []) + _alternative_titles = self.get("alternative_titles", []) - if key == '242__': - _alternative_titles += (alternative_titles_base(self, key, value)) - elif key == '246__': - if ('n' in value and 'p' not in value) or \ - ('n' not in value and 'p' in value): - raise MissingRequiredField(subfield='n or p') + if key == "242__": + _alternative_titles += alternative_titles_base(self, key, value) + elif key == "246__": + if ("n" in value and "p" not in value) or ("n" not in value and "p" in value): + raise MissingRequiredField(subfield="n or p") - if 'p' in value: - _migration = self.get('_migration', {}) - if 'volumes' not in _migration: - _migration['volumes'] = [] + if "p" in value: + _migration = self.get("_migration", {}) + if "volumes" not in _migration: + _migration["volumes"] = [] - val_n = clean_val('n', value, str) - _migration['volumes'].append({ - 'volume': extract_volume_number(val_n, raise_exception=True), - 'title': clean_val('p', value, str), - }) - _migration['is_multipart'] = True - _migration['record_type'] = 'multipart' - self['_migration'] = _migration - raise IgnoreKey('alternative_titles') + val_n = clean_val("n", value, str) + _migration["volumes"].append( + { + "volume": extract_volume_number(val_n, raise_exception=True), + "title": clean_val("p", value, str), + } + ) + _migration["is_multipart"] = True + _migration["record_type"] = "multipart" + self["_migration"] = _migration + raise IgnoreKey("alternative_titles") else: - if 'a' in value: - _alternative_titles.append({ - 'value': clean_val('a', value, str, req=True), - 'type': 'ALTERNATIVE_TITLE', - }) - if 'b' in value: - _alternative_titles.append({ - 'value': clean_val('b', value, str, req=True), - 'type': 'SUBTITLE', - }) + if "a" in value: + _alternative_titles.append( + { + "value": clean_val("a", value, str, req=True), + "type": "ALTERNATIVE_TITLE", + } + ) + if "b" in value: + _alternative_titles.append( + { + "value": clean_val("b", value, str, req=True), + "type": "SUBTITLE", + } + ) return _alternative_titles -@model.over('number_of_pages', '^300__') # item +@model.over("number_of_pages", "^300__") # item def number_of_pages(self, key, value): """Translates number_of_pages fields.""" - val = clean_val('a', value, str) + val = clean_val("a", value, str) if is_excluded(val): - raise IgnoreKey('number_of_pages') + raise IgnoreKey("number_of_pages") parts = extract_parts(val) - if parts['has_extra']: - raise UnexpectedValue(subfield='a') - if parts['physical_copy_description']: - self['physical_copy_description'] = parts['physical_copy_description'] - if parts['number_of_pages']: - return str(parts['number_of_pages']) - raise UnexpectedValue(subfield='a') + if parts["has_extra"]: + raise UnexpectedValue(subfield="a") + if parts["physical_copy_description"]: + self["physical_copy_description"] = parts["physical_copy_description"] + if parts["number_of_pages"]: + return str(parts["number_of_pages"]) + raise UnexpectedValue(subfield="a") -@model.over('title', '^245__') +@model.over("title", "^245__") @out_strip def title(self, key, value): """Translates title.""" - if 'title' in self: + if "title" in self: raise UnexpectedValue() - if 'b' in value: - _alternative_titles = self.get('alternative_titles', []) + if "b" in value: + _alternative_titles = self.get("alternative_titles", []) _alternative_titles.append( - {'value': clean_val('b', value, str), 'type': 'SUBTITLE'}) - self['alternative_titles'] = _alternative_titles - return clean_val('a', value, str, req=True) + {"value": clean_val("b", value, str), "type": "SUBTITLE"} + ) + self["alternative_titles"] = _alternative_titles + return clean_val("a", value, str, req=True) diff --git a/cds_dojson/marc21/fields/books/errors.py b/cds_dojson/marc21/fields/books/errors.py index 7fe9a7c0..70c272c1 100644 --- a/cds_dojson/marc21/fields/books/errors.py +++ b/cds_dojson/marc21/fields/books/errors.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Define Books flavour exceptions.""" + from dojson.errors import DoJSONException @@ -25,8 +26,8 @@ class CDSDoJSONException(DoJSONException): def __init__(self, *args, **kwargs): """Constructor.""" - self.subfield = kwargs.get('subfield', None) - message = kwargs.get('message', None) + self.subfield = kwargs.get("subfield", None) + message = kwargs.get("message", None) if message: self.message = self.message + message @@ -36,16 +37,16 @@ def __init__(self, *args, **kwargs): class UnexpectedValue(CDSDoJSONException): """The corresponding value is unexpected.""" - message = '[UNEXPECTED INPUT VALUE]' + message = "[UNEXPECTED INPUT VALUE]" class MissingRequiredField(CDSDoJSONException): """The corresponding value is required.""" - message = '[MISSING REQUIRED FIELD]' + message = "[MISSING REQUIRED FIELD]" class ManualMigrationRequired(CDSDoJSONException): """The corresponding field should be manually migrated.""" - message = '[MANUAL MIGRATION REQUIRED]' + message = "[MANUAL MIGRATION REQUIRED]" diff --git a/cds_dojson/marc21/fields/books/journal.py b/cds_dojson/marc21/fields/books/journal.py index 47aecc3b..6a8fd6f4 100644 --- a/cds_dojson/marc21/fields/books/journal.py +++ b/cds_dojson/marc21/fields/books/journal.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Journal fields.""" + import re import pycountry @@ -28,124 +29,133 @@ from cds_dojson.marc21.models.books.journal import model -@model.over('legacy_recid', '^001') +@model.over("legacy_recid", "^001") def recid(self, key, value): """Record Identifier.""" return int(value) -@model.over('title', '^245__') +@model.over("title", "^245__") @out_strip def title(self, key, value): """Translates title.""" return base_title(self, key, value) -@model.over('alternative_titles', '^246_3') +@model.over("alternative_titles", "^246_3") @filter_list_values def alternative_titles(self, key, value): """Translates alternative titles.""" - _alternative_titles = self.get('alternative_titles', []) - - if 'a' in value: - _alternative_titles.append({ - 'value': clean_val('a', value, str, req=True), - 'type': 'ALTERNATIVE_TITLE', - }) - if 'b' in value: - _alternative_titles.append({ - 'value': clean_val('b', value, str, req=True), - 'type': 'SUBTITLE', - }) + _alternative_titles = self.get("alternative_titles", []) + + if "a" in value: + _alternative_titles.append( + { + "value": clean_val("a", value, str, req=True), + "type": "ALTERNATIVE_TITLE", + } + ) + if "b" in value: + _alternative_titles.append( + { + "value": clean_val("b", value, str, req=True), + "type": "SUBTITLE", + } + ) return _alternative_titles -@model.over('abbreviated_title', '^210__') +@model.over("abbreviated_title", "^210__") @out_strip def abbreviated_title(self, key, value): """Translates abbreviated title field.""" - return clean_val('a', value, str, req=True) + return clean_val("a", value, str, req=True) -@model.over('identifiers', '^022__') +@model.over("identifiers", "^022__") @filter_list_values @for_each_value def identifiers(self, key, value): """Translates identifiers fields.""" - val_a = clean_val('a', value, str, req=True) + val_a = clean_val("a", value, str, req=True) return { - 'scheme': 'ISSN', - 'value': val_a, - 'material': clean_val('b', value, str), + "scheme": "ISSN", + "value": val_a, + "material": clean_val("b", value, str), } -@model.over('internal_notes', '^937__') +@model.over("internal_notes", "^937__") @for_each_value @out_strip def internal_notes(self, key, value): """Translates internal notes field.""" - return {'value': clean_val('a', value, str, req=True)} + return {"value": clean_val("a", value, str, req=True)} -@model.over('note', '^935__') +@model.over("note", "^935__") @out_strip def note(self, key, value): """Translates note field.""" - return clean_val('a', value, str, req=True) + return clean_val("a", value, str, req=True) -@model.over('publisher', '^933__') +@model.over("publisher", "^933__") @out_strip def publisher(self, key, value): """Translates publisher field.""" - return clean_val('b', value, str, req=True) + return clean_val("b", value, str, req=True) -@model.over('languages', '^041__') +@model.over("languages", "^041__") @for_each_value @out_strip def languages(self, key, value): """Translates languages fields.""" - lang = clean_val('a', value, str).lower() + lang = clean_val("a", value, str).lower() try: return pycountry.languages.lookup(lang).alpha_2.upper() except (KeyError, AttributeError, LookupError): - raise UnexpectedValue(subfield='a') + raise UnexpectedValue(subfield="a") -@model.over('_migration', '(^362__)|(^85641)|(^866__)|(^780__)|(^785__)') +@model.over("_migration", "(^362__)|(^85641)|(^866__)|(^780__)|(^785__)") def migration(self, key, value): """Translates fields related to children record types.""" - _migration = self.get('_migration', {}) - _electronic_items = _migration.get('electronic_items', []) - _items = _migration.get('items', []) - _relation_previous = _migration.get('relation_previous') - _relation_next = _migration.get('relation_next') - if key == '362__': + _migration = self.get("_migration", {}) + _electronic_items = _migration.get("electronic_items", []) + _items = _migration.get("items", []) + _relation_previous = _migration.get("relation_previous") + _relation_next = _migration.get("relation_next") + if key == "362__": + _electronic_items.append({"subscription": clean_val("a", value, str)}) + if key == "85641": _electronic_items.append( - {'subscription': clean_val('a', value, str)} + { + "subscription": clean_val("3", value, str), + "url": clean_val("u", value, str), + "access_type": clean_val("x", value, str), + "note": clean_val("z", value, str), + } ) - if key == '85641': - _electronic_items.append( - {'subscription': clean_val('3', value, str), - 'url': clean_val('u', value, str), - 'access_type': clean_val('x', value, str), - 'note': clean_val('z', value, str), - } + if key == "866__": + _items.append( + { + "subscription": clean_val("a", value, str), + } ) - if key == '866__': - _items.append({ - 'subscription': clean_val('a', value, str), - }) - if key == '780__': - _relation_previous = clean_val('w', value, str, req=True) - - if key == '785__': - _relation_next = clean_val('w', value, str, req=True) - - _migration.update({'electronic_items': _electronic_items, 'items': _items, - 'relation_previous': _relation_previous, - 'relation_next': _relation_next - }) + if key == "780__": + _relation_previous = clean_val("w", value, str, req=True) + + if key == "785__": + _relation_next = clean_val("w", value, str, req=True) + + _migration.update( + { + "electronic_items": _electronic_items, + "items": _items, + "relation_previous": _relation_previous, + "relation_next": _relation_next, + } + ) return _migration diff --git a/cds_dojson/marc21/fields/books/multipart.py b/cds_dojson/marc21/fields/books/multipart.py index a9417c42..6d95c516 100644 --- a/cds_dojson/marc21/fields/books/multipart.py +++ b/cds_dojson/marc21/fields/books/multipart.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Books fields.""" + import re from copy import deepcopy @@ -44,41 +45,41 @@ def _insert_volume(_migration, volume_number, volume_obj): """Finds the corresponding volume, or create it, and insert the attribute.""" - volumes = _migration['volumes'] + volumes = _migration["volumes"] volume_obj = deepcopy(volume_obj) - volume_obj['volume'] = volume_number + volume_obj["volume"] = volume_number volumes.append(volume_obj) return volume_obj -@model.over('identifiers', '^020__', override=True) +@model.over("identifiers", "^020__", override=True) @filter_list_values @for_each_value def isbns(self, key, value): """Translates isbns stored in the record.""" - _migration = self['_migration'] - _identifiers = self.get('identifiers', []) + _migration = self["_migration"] + _identifiers = self.get("identifiers", []) - val_u = clean_val('u', value, str) - val_a = clean_val('a', value, str) - val_b = clean_val('b', value, str) + val_u = clean_val("u", value, str) + val_a = clean_val("a", value, str) + val_b = clean_val("b", value, str) if val_u: volume_info = extract_volume_info(val_u) # if set found it means that the isbn is for the whole multipart - set_search = re.search('(.*?)\(set\.*\)', val_u) + set_search = re.search(r"(.*?)\(set\.*\)", val_u) if volume_info: # if we have volume there it means that the ISBN is of the volume volume_obj = { - 'isbn': clean_val('a', value, str), - 'physical_description': volume_info['description'].strip(), - 'is_electronic': val_b is not None, + "isbn": clean_val("a", value, str), + "physical_description": volume_info["description"].strip(), + "is_electronic": val_b is not None, } - _insert_volume(_migration, volume_info['volume'], volume_obj) - raise IgnoreKey('identifiers') + _insert_volume(_migration, volume_info["volume"], volume_obj) + raise IgnoreKey("identifiers") if set_search: - self['physical_description'] = set_search.group(1).strip() - isbn = {'scheme': 'ISBN', 'value': val_a} + self["physical_description"] = set_search.group(1).strip() + isbn = {"scheme": "ISBN", "value": val_a} return isbn if isbn not in _identifiers else None if not volume_info: # Try to find a volume number @@ -86,258 +87,265 @@ def isbns(self, key, value): if volume_number: # volume, but without description volume_obj = { - 'isbn': clean_val('a', value, str), - 'is_electronic': val_b is not None, + "isbn": clean_val("a", value, str), + "is_electronic": val_b is not None, } _insert_volume(_migration, volume_number, volume_obj) - raise IgnoreKey('identifiers') + raise IgnoreKey("identifiers") elif extract_volume_number(val_u, search=True): raise UnexpectedValue( - subfield='u', - message=' found volume but failed to parse description' + subfield="u", + message=" found volume but failed to parse description", ) else: - self['physical_description'] = val_u - isbn = {'scheme': 'ISBN', 'value': val_a} + self["physical_description"] = val_u + isbn = {"scheme": "ISBN", "value": val_a} return isbn if isbn not in _identifiers else None if not set_search and not volume_info: - self['physical_description'] = val_u - isbn = {'scheme': 'ISBN', 'value': val_a} + self["physical_description"] = val_u + isbn = {"scheme": "ISBN", "value": val_a} return isbn if isbn not in _identifiers else None elif not val_u and val_a: # if I dont have volume info but only isbn - isbn = {'scheme': 'ISBN', 'value': val_a} + isbn = {"scheme": "ISBN", "value": val_a} return isbn if isbn not in _identifiers else None else: - raise UnexpectedValue(subfield='a', message=' isbn not provided') + raise UnexpectedValue(subfield="a", message=" isbn not provided") -@model.over('dois', '^0247_', override=True) +@model.over("dois", "^0247_", override=True) @filter_list_values def dois(self, key, value): """Translates DOIs.""" - _migration = self['_migration'] - _identifiers = self.get('identifiers', []) + _migration = self["_migration"] + _identifiers = self.get("identifiers", []) for v in force_list(value): - val_2 = clean_val('2', v, str) - if val_2 and val_2 != 'DOI': - raise UnexpectedValue(subfield='2', message=' field is not equal to DOI') - val_q = clean_val('q', v, str, transform='lower') + val_2 = clean_val("2", v, str) + if val_2 and val_2 != "DOI": + raise UnexpectedValue(subfield="2", message=" field is not equal to DOI") + val_q = clean_val("q", v, str, transform="lower") volume_info = extract_volume_info(val_q) doi = { - 'value': clean_val('a', v, str, req=True), - 'source': clean_val('9', v, str), - 'scheme': 'DOI', + "value": clean_val("a", v, str, req=True), + "source": clean_val("9", v, str), + "scheme": "DOI", } if volume_info: # this identifier is for a specific volume volume_obj = { - 'doi': doi['value'], - 'material': mapping(MATERIALS, volume_info['description'], raise_exception=True), - 'source': doi['source'] + "doi": doi["value"], + "material": mapping( + MATERIALS, volume_info["description"], raise_exception=True + ), + "source": doi["source"], } - _insert_volume(_migration, volume_info['volume'], volume_obj) + _insert_volume(_migration, volume_info["volume"], volume_obj) else: - if re.match(r'.* \(.*\)', val_q): - raise UnexpectedValue(subfield='q', - message=' found a volume number but could not extract it') - doi['material'] = mapping(MATERIALS, val_q, raise_exception=True) + if re.match(r".* \(.*\)", val_q): + raise UnexpectedValue( + subfield="q", + message=" found a volume number but could not extract it", + ) + doi["material"] = mapping(MATERIALS, val_q, raise_exception=True) if doi not in _identifiers: _identifiers.append(doi) if len(_identifiers) > 0: - self['identifiers'] = _identifiers + self["identifiers"] = _identifiers -@model.over('alternative_identifiers', '(^035__)|(^036__)', override=True) +@model.over("alternative_identifiers", "(^035__)|(^036__)", override=True) def alternative_identifiers(self, key, value): """Translates external_system_identifiers fields.""" return alternative_identifiers_base(self, key, value) -@model.over('barcode', '^088__', override=True) +@model.over("barcode", "^088__", override=True) def barcode(self, key, value): """Translates the barcodes.""" - _migration = self['_migration'] + _migration = self["_migration"] for v in force_list(value): - val_a = clean_val('a', v, str) - val_n = clean_val('n', v, str) - val_x = clean_val('x', v, str) - val_9 = clean_val('9', v, str) + val_a = clean_val("a", v, str) + val_n = clean_val("n", v, str) + val_x = clean_val("x", v, str) + val_9 = clean_val("9", v, str) if val_a or val_9: if val_n or val_x or val_a and val_9: raise UnexpectedValue() - identifier = {'scheme': 'report_number', 'value': val_a or val_9} + identifier = {"scheme": "report_number", "value": val_a or val_9} if val_9: - identifier['hidden'] = True - identifiers = self.get('identifiers', []) + identifier["hidden"] = True + identifiers = self.get("identifiers", []) identifiers.append(identifier) - self['identifiers'] = identifiers - raise IgnoreKey('barcode') + self["identifiers"] = identifiers + raise IgnoreKey("barcode") if val_n and val_x: volume_number = extract_volume_number( - val_n, - raise_exception=True, - subfield='n' + val_n, raise_exception=True, subfield="n" ) - _insert_volume(_migration, volume_number, {'barcode': val_x}) + _insert_volume(_migration, volume_number, {"barcode": val_x}) elif val_x: - raise MissingRequiredField(subfield='n', - message=' this record is missing a volume number') + raise MissingRequiredField( + subfield="n", message=" this record is missing a volume number" + ) else: - raise MissingRequiredField(subfield='x', - message=' this record is missing a barcode number') - raise IgnoreKey('barcode') + raise MissingRequiredField( + subfield="x", message=" this record is missing a barcode number" + ) + raise IgnoreKey("barcode") -@model.over('authors', '(^100__)|(^700__)', override=True) +@model.over("authors", "(^100__)|(^700__)", override=True) def authors(self, key, value): """Translates the authors field.""" - _migration = self['_migration'] - _authors = _migration.get('authors', []) + _migration = self["_migration"] + _authors = _migration.get("authors", []) item = build_contributor_books(value) if item and item not in _authors: _authors.append(item) try: - if 'u' in value: - other = ['et al.', 'et al'] - val_u = list(force_list(value.get('u'))) + if "u" in value: + other = ["et al.", "et al"] + val_u = list(force_list(value.get("u"))) if [i for i in other if i in val_u]: - _migration['other_authors'] = True + _migration["other_authors"] = True except UnexpectedValue: pass - _migration['authors'] = _authors - return list(map(lambda author: author['full_name'], _authors)) + _migration["authors"] = _authors + return list(map(lambda author: author["full_name"], _authors)) -@model.over('title', '^245__') +@model.over("title", "^245__") @out_strip def title(self, key, value): """Translates book series title.""" # assume that is goes by order of fields and check 245 first - if 'b' in value: - _alternative_titles = self.get('alternative_titles', []) - _alternative_titles.append({'type': 'SUBTITLE', - 'value': clean_val('b', value, str)}) - self['alternative_titles'] = _alternative_titles - return clean_val('a', value, str) + if "b" in value: + _alternative_titles = self.get("alternative_titles", []) + _alternative_titles.append( + {"type": "SUBTITLE", "value": clean_val("b", value, str)} + ) + self["alternative_titles"] = _alternative_titles + return clean_val("a", value, str) -@model.over('_migration', '^246__') +@model.over("_migration", "^246__") def migration(self, key, value): """Translates volumes titles.""" - volume_title = self.get('title', None) - _migration = self['_migration'] + volume_title = self.get("title", None) + _migration = self["_migration"] for v in force_list(value): # check if it is a multipart monograph - val_n = clean_val('n', v, str) - val_p = clean_val('p', v, str) - val_y = clean_val('y', v, str) + val_n = clean_val("n", v, str) + val_p = clean_val("p", v, str) + val_y = clean_val("y", v, str) if not val_n and not val_p: raise UnexpectedValue( - subfield='n', message=' this record is probably not a series') + subfield="n", message=" this record is probably not a series" + ) if val_p and not val_n: raise UnexpectedValue( - subfield='n', message=' volume title exists but no volume number' + subfield="n", message=" volume title exists but no volume number" ) - volume_index = re.findall(r'\d+', val_n) if val_n else None + volume_index = re.findall(r"\d+", val_n) if val_n else None if volume_index and len(volume_index) > 1: - raise UnexpectedValue(subfield='n', - message=' volume has more than one digit ') + raise UnexpectedValue( + subfield="n", message=" volume has more than one digit " + ) else: volume_number = extract_volume_number( - val_n, - raise_exception=True, - subfield='n' + val_n, raise_exception=True, subfield="n" ) - obj = {'title': val_p or volume_title} + obj = {"title": val_p or volume_title} if val_y: - if re.match('\\d+', val_y) and 1600 <= int(val_y) <= 2021: - obj['publication_year'] = int(val_y) + if re.match("\\d+", val_y) and 1600 <= int(val_y) <= 2021: + obj["publication_year"] = int(val_y) else: - raise UnexpectedValue(subfield='y', - message=' unrecognized publication year') + raise UnexpectedValue( + subfield="y", message=" unrecognized publication year" + ) _insert_volume(_migration, volume_number, obj) if not volume_title: raise MissingRequiredField( - subfield='a', message=' this record is missing a main title') + subfield="a", message=" this record is missing a main title" + ) # series created return _migration -@model.over('number_of_volumes', '^300__') +@model.over("number_of_volumes", "^300__") @out_strip def number_of_volumes(self, key, value): """Translates number of volumes.""" - _series_title = self.get('title', None) + _series_title = self.get("title", None) if not _series_title: raise MissingRequiredField( - subfield='a', message=' this record is missing a main title') - val_a = clean_val('a', value, str) + subfield="a", message=" this record is missing a main title" + ) + val_a = clean_val("a", value, str) parsed_a = extract_parts(val_a) - if not parsed_a["number_of_pages"] and ('v' in val_a or 'vol' in val_a): - _volumes = re.findall(r'\d+', val_a) + if not parsed_a["number_of_pages"] and ("v" in val_a or "vol" in val_a): + _volumes = re.findall(r"\d+", val_a) if _volumes: return _volumes[0] - raise IgnoreKey('number_of_volumes') + raise IgnoreKey("number_of_volumes") -@model.over('multivolume_record_format', '^596__') +@model.over("multivolume_record_format", "^596__") def multivolume_record_format(self, key, value): """Multivolume kind.""" - val_a = clean_val('a', value, str) - _migration = self['_migration'] - if val_a == 'MULTIVOLUMES-1': + val_a = clean_val("a", value, str) + _migration = self["_migration"] + if val_a == "MULTIVOLUMES-1": parsed = True - elif val_a == 'MULTIVOLUMES-X' or val_a == 'MULTIVOLUMES-x': + elif val_a == "MULTIVOLUMES-X" or val_a == "MULTIVOLUMES-x": parsed = False - elif val_a == 'MULTIVOLUMES-MANUAL': - raise Exception('This record should not be migrated!') + elif val_a == "MULTIVOLUMES-MANUAL": + raise Exception("This record should not be migrated!") else: raise UnexpectedValue( - subfield='a', message=' unrecognized migration multipart tag' + subfield="a", message=" unrecognized migration multipart tag" ) - _migration['multivolume_record_format'] = parsed - raise IgnoreKey('multivolume_record_format') + _migration["multivolume_record_format"] = parsed + raise IgnoreKey("multivolume_record_format") -@model.over('multipart_id', '^597__') +@model.over("multipart_id", "^597__") def multipart_id(self, key, value): """Volume serial id.""" - val_a = clean_val('a', value, str) - _migration = self['_migration'] - _migration['multipart_id'] = val_a - raise IgnoreKey('multipart_id') + val_a = clean_val("a", value, str) + _migration = self["_migration"] + _migration["multipart_id"] = val_a + raise IgnoreKey("multipart_id") -@model.over('urls', '^8564_', override=True) +@model.over("urls", "^8564_", override=True) def urls(self, key, value): """Translates urls field.""" - sub_y = clean_val('y', value, str, default='') - sub_u = clean_val('u', value, str, req=True) + sub_y = clean_val("y", value, str, default="") + sub_u = clean_val("u", value, str, req=True) - _migration = self['_migration'] + _migration = self["_migration"] volume_info = extract_volume_info(sub_y) if sub_y else None if volume_info: # url for a specific volume # TODO? - description = volume_info['description'] - volume_number = volume_info['volume'] - if description != 'ebook': - raise UnexpectedValue(subfield='y', - message=' unsupported value') + description = volume_info["description"] + volume_number = volume_info["volume"] + if description != "ebook": + raise UnexpectedValue(subfield="y", message=" unsupported value") volume_obj = { - 'url': sub_u, - 'description': description, + "url": sub_u, + "description": description, } - _insert_volume(_migration, volume_info['volume'], volume_obj) - raise IgnoreKey('urls') + _insert_volume(_migration, volume_info["volume"], volume_obj) + raise IgnoreKey("urls") else: return urls_base(self, key, value) diff --git a/cds_dojson/marc21/fields/books/serial.py b/cds_dojson/marc21/fields/books/serial.py index 3770f789..e103c5d9 100644 --- a/cds_dojson/marc21/fields/books/serial.py +++ b/cds_dojson/marc21/fields/books/serial.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Books fields.""" + from dojson.utils import for_each_value from cds_dojson.marc21.fields.books.multipart import isbns as multipart_identifiers @@ -24,27 +25,27 @@ from cds_dojson.marc21.models.books.serial import model -@model.over('legacy_recid', '^001') +@model.over("legacy_recid", "^001") def recid(self, key, value): """Record Identifier.""" return int(value) -@model.over('title', '^490__') +@model.over("title", "^490__") @for_each_value @out_strip def title(self, key, value): """Translates book series title.""" - _identifiers = self.get('identifiers', []) - issn = clean_val('x', value, str) + _identifiers = self.get("identifiers", []) + issn = clean_val("x", value, str) if issn: - _identifiers.append({'scheme': 'ISSN', 'value': issn}) - self['identifiers'] = _identifiers - self['mode_of_issuance'] = 'SERIAL' - return clean_val('a', value, str, req=True) + _identifiers.append({"scheme": "ISSN", "value": issn}) + self["identifiers"] = _identifiers + self["mode_of_issuance"] = "SERIAL" + return clean_val("a", value, str, req=True) -@model.over('identifiers', '^020__') +@model.over("identifiers", "^020__") @filter_list_values def identifiers(self, key, value): """Translates identifiers fields.""" diff --git a/cds_dojson/marc21/fields/books/standard.py b/cds_dojson/marc21/fields/books/standard.py index 2fc6b2e4..7b7f36ae 100644 --- a/cds_dojson/marc21/fields/books/standard.py +++ b/cds_dojson/marc21/fields/books/standard.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Standards fields.""" + from __future__ import unicode_literals from dojson.errors import IgnoreKey @@ -27,54 +28,59 @@ from cds_dojson.marc21.models.books.standard import model -@model.over('alternative_titles', '^246__') +@model.over("alternative_titles", "^246__") @filter_list_values def title_translations(self, key, value): """Translates title translations.""" - _alternative_titles = self.get('alternative_titles', []) + _alternative_titles = self.get("alternative_titles", []) - if 'a' in value: - _alternative_titles.append({ - 'value': clean_val('a', value, str, req=True), - 'type': 'TRANSLATED_TITLE', - 'language': 'fr', - }) - if 'b' in value: - _alternative_titles.append({ - 'value': clean_val('b', value, str, req=True), - 'type': 'TRANSLATED_SUBTITLE', - 'language': 'fr', - }) + if "a" in value: + _alternative_titles.append( + { + "value": clean_val("a", value, str, req=True), + "type": "TRANSLATED_TITLE", + "language": "fr", + } + ) + if "b" in value: + _alternative_titles.append( + { + "value": clean_val("b", value, str, req=True), + "type": "TRANSLATED_SUBTITLE", + "language": "fr", + } + ) return _alternative_titles -@model.over('number_of_pages', '^300__') # item +@model.over("number_of_pages", "^300__") # item def number_of_pages(self, key, value): """Translates number_of_pages fields.""" - val = clean_val('a', value, str) + val = clean_val("a", value, str) if is_excluded(val): - raise IgnoreKey('number_of_pages') + raise IgnoreKey("number_of_pages") parts = extract_parts(val) - if parts['has_extra']: - raise UnexpectedValue(subfield='a') - if parts['physical_copy_description']: - self['physical_copy_description'] = parts['physical_copy_description'] - if parts['number_of_pages']: - return parts['number_of_pages'] - raise UnexpectedValue(subfield='a') + if parts["has_extra"]: + raise UnexpectedValue(subfield="a") + if parts["physical_copy_description"]: + self["physical_copy_description"] = parts["physical_copy_description"] + if parts["number_of_pages"]: + return parts["number_of_pages"] + raise UnexpectedValue(subfield="a") -@model.over('title', '^245__') +@model.over("title", "^245__") @out_strip def title(self, key, value): """Translates title.""" - if 'title' in self: + if "title" in self: raise UnexpectedValue() - if 'b' in value: - _alternative_titles = self.get('alternative_titles', []) + if "b" in value: + _alternative_titles = self.get("alternative_titles", []) _alternative_titles.append( - {'value': clean_val('b', value, str), 'type': 'SUBTITLE'}) - self['alternative_titles'] = _alternative_titles - return clean_val('a', value, str, req=True) + {"value": clean_val("b", value, str), "type": "SUBTITLE"} + ) + self["alternative_titles"] = _alternative_titles + return clean_val("a", value, str, req=True) diff --git a/cds_dojson/marc21/fields/books/utils.py b/cds_dojson/marc21/fields/books/utils.py index 27e6609f..beb9c022 100644 --- a/cds_dojson/marc21/fields/books/utils.py +++ b/cds_dojson/marc21/fields/books/utils.py @@ -24,14 +24,22 @@ MAX_PAGES_NUMBER = 8192 -RE_STR_VOLUME_PREFIX = r'(?:(?:[Vv](?:ol(?:ume)?)?|[Pp](?:art(?:ie)?|t)?|[Tt](?:eil)?|[Bb]d|[Tt]ome?|course|conference|fasc(?:icule)?|book|unit|suppl|Tafeln|Tomo)[\s\.]*)' -RE_STR_ROMAN_NUMERAL = r'(?:(?:IX|IV|V?I{1,3})|X{1,3}(?:IX|IV|V?I{0,3}))' -RE_STR_VOLUME_SUFFIX = r'(\d{{1,4}}|\d\d?[a-zA-Z]|[a-zA-Z]\d|[A-H]|{})'.format(RE_STR_ROMAN_NUMERAL) -RE_STR_VOLUME = r'(?:{}?{})'.format(RE_STR_VOLUME_PREFIX, RE_STR_VOLUME_SUFFIX) -RE_STR_SPECIAL = r'[^0-9A-Za-zÀ-ÿ\-/]' - -RE_VOLUME_NUMBER = re.compile(r'(?:^|{}){}(?:$|{})'.format(RE_STR_SPECIAL, RE_STR_VOLUME, RE_STR_SPECIAL)) -RE_VOLUME_INFO = re.compile(r'(.*?)\({}\)'.format(RE_STR_VOLUME)) +RE_STR_VOLUME_PREFIX = ( + r"(?:(?:[Vv](?:ol(?:ume)?)?|[Pp](?:art(?:ie)?|t)?|[Tt](?:eil)?|[Bb]d|" + r"[Tt]ome?|course|conference|fasc(?:icule)?|book|unit|suppl|Tafeln|Tomo)" + r"[\s\.]*)" +) +RE_STR_ROMAN_NUMERAL = r"(?:(?:IX|IV|V?I{1,3})|X{1,3}(?:IX|IV|V?I{0,3}))" +RE_STR_VOLUME_SUFFIX = r"(\d{{1,4}}|\d\d?[a-zA-Z]|[a-zA-Z]\d|[A-H]|{})".format( + RE_STR_ROMAN_NUMERAL +) +RE_STR_VOLUME = r"(?:{}?{})".format(RE_STR_VOLUME_PREFIX, RE_STR_VOLUME_SUFFIX) +RE_STR_SPECIAL = r"[^0-9A-Za-zÀ-ÿ\-/]" + +RE_VOLUME_NUMBER = re.compile( + r"(?:^|{}){}(?:$|{})".format(RE_STR_SPECIAL, RE_STR_VOLUME, RE_STR_SPECIAL) +) +RE_VOLUME_INFO = re.compile(r"(.*?)\({}\)".format(RE_STR_VOLUME)) def is_excluded(value): @@ -63,9 +71,7 @@ def extract_number_of_pages(value): def extract_physical_description(value): """Extract physical description from 300 if any.""" res = re.findall( - r"([0-9]+ \w[CD\-ROM|DVD\-ROM|diskette|VHS]+)", - value, - flags=re.IGNORECASE + r"([0-9]+ \w[CD\-ROM|DVD\-ROM|diskette|VHS]+)", value, flags=re.IGNORECASE ) if res: return ", ".join(res).upper() @@ -88,7 +94,7 @@ def extract_parts(value): physical_description = extract_physical_description(value) if physical_description: - valid_parts_count -= len(physical_description.split(',')) + valid_parts_count -= len(physical_description.split(",")) return { "has_extra": bool(valid_parts_count > 0), @@ -97,8 +103,7 @@ def extract_parts(value): } -def extract_volume_number(value, search=False, raise_exception=False, - subfield=None): +def extract_volume_number(value, search=False, raise_exception=False, subfield=None): """Extract the volume number from a string, returns None if not matched.""" regex = RE_VOLUME_NUMBER if search: @@ -112,7 +117,8 @@ def extract_volume_number(value, search=False, raise_exception=False, if raise_exception: raise MissingRequiredField( - subfield=subfield, message=' failed to parse volume number') + subfield=subfield, message=" failed to parse volume number" + ) return None diff --git a/cds_dojson/marc21/fields/books/values_mapping.py b/cds_dojson/marc21/fields/books/values_mapping.py index 07b00931..84beed9e 100644 --- a/cds_dojson/marc21/fields/books/values_mapping.py +++ b/cds_dojson/marc21/fields/books/values_mapping.py @@ -17,136 +17,262 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Book values mapping.""" + from __future__ import unicode_literals from cds_dojson.marc21.fields.books.errors import UnexpectedValue DOCUMENT_TYPE = { - 'PROCEEDINGS': ['PROCEEDINGS', '42', '43'], - 'BOOK': ['BOOK', '21'], - 'STANDARD': ['STANDARD'] + "PROCEEDINGS": ["PROCEEDINGS", "42", "43"], + "BOOK": ["BOOK", "21"], + "STANDARD": ["STANDARD"], } COLLECTION = { - 'BOOK SUGGESTION': ['BOOKSUGGESTION'], - 'LEGSERLIB': ['LEGSERLIB'], - 'YELLOW REPORT': ['YELLOW REPORT', 'YELLOWREPORT'], - 'CERN': ['CERN'], - 'DESIGN REPORT': ['DESIGN REPORT', 'DESIGNREPORT'], - 'BOOKSHOP': ['BOOKSHOP'], - 'LEGSERLIBINTLAW': ['LEGSERLIBINTLAW'], - 'LEGSERLIBCIVLAW': ['LEGSERLIBCIVLAW'], - 'LEGSERLIBLEGRES': ['LEGSERLIBLEGRES'] + "BOOK SUGGESTION": ["BOOKSUGGESTION"], + "LEGSERLIB": ["LEGSERLIB"], + "YELLOW REPORT": ["YELLOW REPORT", "YELLOWREPORT"], + "CERN": ["CERN"], + "DESIGN REPORT": ["DESIGN REPORT", "DESIGNREPORT"], + "BOOKSHOP": ["BOOKSHOP"], + "LEGSERLIBINTLAW": ["LEGSERLIBINTLAW"], + "LEGSERLIBCIVLAW": ["LEGSERLIBCIVLAW"], + "LEGSERLIBLEGRES": ["LEGSERLIBLEGRES"], } ACQUISITION_METHOD = { # mapping for possible value of "created_by.type" field - 'user': ['H', 'r'], - 'batchuploader': ['N', 'M'], - 'migration': ['migration'], + "user": ["H", "r"], + "batchuploader": ["N", "M"], + "migration": ["migration"], } MEDIUM_TYPES = [ - 'ELECTRONIC VERSION', - 'PRINT VERSION', - 'PRINT VERSION, HARDBACK', - 'PRINT VERSION, PAPERBACK', - 'PRINT VERSION, SPIRAL-BOUND', - 'CD-ROM', - 'AUDIOBOOK', - 'DVD', + "ELECTRONIC VERSION", + "PRINT VERSION", + "PRINT VERSION, HARDBACK", + "PRINT VERSION, PAPERBACK", + "PRINT VERSION, SPIRAL-BOUND", + "CD-ROM", + "AUDIOBOOK", + "DVD", ] -ARXIV_CATEGORIES = ['astro-ph', 'astro-ph.CO', 'astro-ph.EP', 'astro-ph.GA', - 'astro-ph.HE', 'astro-ph.IM', 'astro-ph.SR', 'cond-mat', - 'cond-mat.dis-nn', 'cond-mat.mes-hall', - 'cond-mat.mtrl-sci', 'cond-mat.other', - 'cond-mat.quant-gas', 'cond-mat.soft', - 'cond-mat.stat-mech', 'cond-mat.str-el', - 'cond-mat.supr-con', 'cs', 'cs.AI', 'cs.AR', 'cs.CC', - 'cs.CE', 'cs.CG', 'cs.CL', 'cs.CR', 'cs.CV', 'cs.CY', - 'cs.DB', 'cs.DC', 'cs.DL', 'cs.DM', 'cs.DS', 'cs.ET', - 'cs.FL', 'cs.GL', 'cs.GR', 'cs.GT', 'cs.HC', 'cs.IR', - 'cs.IT', 'cs.LG', 'cs.LO', 'cs.MA', 'cs.MM', 'cs.MS', - 'cs.NA', 'cs.NE', 'cs.NI', 'cs.OH', 'cs.OS', 'cs.PF', - 'cs.PL', 'cs.RO', 'cs.SC', 'cs.SD', 'cs.SE', 'cs.SI', - 'cs.SY', 'econ', 'econ.EM', 'eess', 'eess.AS', 'eess.IV', - 'eess.SP', 'gr-qc', 'hep-ex', 'hep-lat', 'hep-ph', - 'hep-th', 'math', 'math-ph', 'math.AC', 'math.AG', - 'math.AP', 'math.AT', 'math.CA', 'math.CO', 'math.CT', - 'math.CV', 'math.DG', 'math.DS', 'math.FA', 'math.GM', - 'math.GN', 'math.GR', 'math.GT', 'math.HO', 'math.IT', - 'math.KT', 'math.LO', 'math.MG', 'math.MP', 'math.NA', - 'math.NT', 'math.OA', 'math.OC', 'math.PR', 'math.QA', - 'math.RA', 'math.RT', 'math.SG', 'math.SP', 'math.ST', - 'nlin', 'nlin.AO', 'nlin.CD', 'nlin.CG', 'nlin.PS', - 'nlin.SI', 'nucl-ex', 'nucl-th', 'physics', - 'physics.acc-ph', 'physics.ao-ph', 'physics.app-ph', - 'physics.atm-clus', 'physics.atom-ph', 'physics.bio-ph', - 'physics.chem-ph', 'physics.class-ph', 'physics.comp-ph', - 'physics.data-an', 'physics.ed-ph', 'physics.flu-dyn', - 'physics.gen-ph', 'physics.geo-ph', 'physics.hist-ph', - 'physics.ins-det', 'physics.med-ph', 'physics.optics', - 'physics.plasm-ph', 'physics.pop-ph', 'physics.soc-ph', - 'physics.space-ph', 'q-bio', 'q-bio.BM', 'q-bio.CB', - 'q-bio.GN', 'q-bio.MN', 'q-bio.NC', 'q-bio.OT', 'q-bio.PE', - 'q-bio.QM', 'q-bio.SC', 'q-bio.TO', 'q-fin', 'q-fin.CP', - 'q-fin.EC', 'q-fin.GN', 'q-fin.MF', 'q-fin.PM', 'q-fin.PR', - 'q-fin.RM', 'q-fin.ST', 'q-fin.TR', 'quant-ph', 'stat', - 'stat.AP', 'stat.CO', 'stat.ME', 'stat.ML', 'stat.OT', - 'stat.TH'] +ARXIV_CATEGORIES = [ + "astro-ph", + "astro-ph.CO", + "astro-ph.EP", + "astro-ph.GA", + "astro-ph.HE", + "astro-ph.IM", + "astro-ph.SR", + "cond-mat", + "cond-mat.dis-nn", + "cond-mat.mes-hall", + "cond-mat.mtrl-sci", + "cond-mat.other", + "cond-mat.quant-gas", + "cond-mat.soft", + "cond-mat.stat-mech", + "cond-mat.str-el", + "cond-mat.supr-con", + "cs", + "cs.AI", + "cs.AR", + "cs.CC", + "cs.CE", + "cs.CG", + "cs.CL", + "cs.CR", + "cs.CV", + "cs.CY", + "cs.DB", + "cs.DC", + "cs.DL", + "cs.DM", + "cs.DS", + "cs.ET", + "cs.FL", + "cs.GL", + "cs.GR", + "cs.GT", + "cs.HC", + "cs.IR", + "cs.IT", + "cs.LG", + "cs.LO", + "cs.MA", + "cs.MM", + "cs.MS", + "cs.NA", + "cs.NE", + "cs.NI", + "cs.OH", + "cs.OS", + "cs.PF", + "cs.PL", + "cs.RO", + "cs.SC", + "cs.SD", + "cs.SE", + "cs.SI", + "cs.SY", + "econ", + "econ.EM", + "eess", + "eess.AS", + "eess.IV", + "eess.SP", + "gr-qc", + "hep-ex", + "hep-lat", + "hep-ph", + "hep-th", + "math", + "math-ph", + "math.AC", + "math.AG", + "math.AP", + "math.AT", + "math.CA", + "math.CO", + "math.CT", + "math.CV", + "math.DG", + "math.DS", + "math.FA", + "math.GM", + "math.GN", + "math.GR", + "math.GT", + "math.HO", + "math.IT", + "math.KT", + "math.LO", + "math.MG", + "math.MP", + "math.NA", + "math.NT", + "math.OA", + "math.OC", + "math.PR", + "math.QA", + "math.RA", + "math.RT", + "math.SG", + "math.SP", + "math.ST", + "nlin", + "nlin.AO", + "nlin.CD", + "nlin.CG", + "nlin.PS", + "nlin.SI", + "nucl-ex", + "nucl-th", + "physics", + "physics.acc-ph", + "physics.ao-ph", + "physics.app-ph", + "physics.atm-clus", + "physics.atom-ph", + "physics.bio-ph", + "physics.chem-ph", + "physics.class-ph", + "physics.comp-ph", + "physics.data-an", + "physics.ed-ph", + "physics.flu-dyn", + "physics.gen-ph", + "physics.geo-ph", + "physics.hist-ph", + "physics.ins-det", + "physics.med-ph", + "physics.optics", + "physics.plasm-ph", + "physics.pop-ph", + "physics.soc-ph", + "physics.space-ph", + "q-bio", + "q-bio.BM", + "q-bio.CB", + "q-bio.GN", + "q-bio.MN", + "q-bio.NC", + "q-bio.OT", + "q-bio.PE", + "q-bio.QM", + "q-bio.SC", + "q-bio.TO", + "q-fin", + "q-fin.CP", + "q-fin.EC", + "q-fin.GN", + "q-fin.MF", + "q-fin.PM", + "q-fin.PR", + "q-fin.RM", + "q-fin.ST", + "q-fin.TR", + "quant-ph", + "stat", + "stat.AP", + "stat.CO", + "stat.ME", + "stat.ML", + "stat.OT", + "stat.TH", +] MATERIALS = [ - 'addendum', - 'additional material', - 'data', - 'e-proceedings', - 'ebook', - 'editorial note', - 'erratum', - 'preprint', - 'publication', - 'reprint', - 'software', - 'translation', + "addendum", + "additional material", + "data", + "e-proceedings", + "ebook", + "editorial note", + "erratum", + "preprint", + "publication", + "reprint", + "software", + "translation", ] -SUBJECT_CLASSIFICATION_EXCEPTIONS = \ - ['PACS', 'CERN LIBRARY', 'CERN YELLOW REPORT'] +SUBJECT_CLASSIFICATION_EXCEPTIONS = ["PACS", "CERN LIBRARY", "CERN YELLOW REPORT"] -AUTHOR_AFFILIATION_SCHEMES = ['ROR', 'GRID'] +AUTHOR_AFFILIATION_SCHEMES = ["ROR", "GRID"] EXTERNAL_SYSTEM_IDENTIFIERS = [ - 'ARXIV', - 'EBL', - 'INSPIRE', - 'SAFARI', - 'SCEM', - 'HDL', + "ARXIV", + "EBL", + "INSPIRE", + "SAFARI", + "SCEM", + "HDL", ] EXTERNAL_SYSTEM_IDENTIFIERS_TO_IGNORE = [ - 'ASIN', - 'DCL', - 'DESY', - 'DOE', - 'ASIN' - 'IEECONF', - 'CERN ANNUAL REPORT', - 'HTTP://INSPIREHEP.NET/OAI2D', - 'FIZ', - 'HAL', - 'KEK', - 'INIS', - 'SLAC', - 'LHCLHC', - 'INDICO.CERN.CH', - 'UDCCERN', - 'SLACCONF', - 'SPIRES', - 'WAI01', - 'URN' + "ASIN", + "DCL", + "DESY", + "DOE", + "ASIN" "IEECONF", + "CERN ANNUAL REPORT", + "HTTP://INSPIREHEP.NET/OAI2D", + "FIZ", + "HAL", + "KEK", + "INIS", + "SLAC", + "LHCLHC", + "INDICO.CERN.CH", + "UDCCERN", + "SLACCONF", + "SPIRES", + "WAI01", + "URN", ] diff --git a/cds_dojson/marc21/fields/rdm/base.py b/cds_dojson/marc21/fields/rdm/base.py index a29a28f3..b54e2454 100644 --- a/cds_dojson/marc21/fields/rdm/base.py +++ b/cds_dojson/marc21/fields/rdm/base.py @@ -26,16 +26,16 @@ from cds_dojson.marc21.models.base import model -@model.over('preprint_date', '^269__') # item, RDM?! +@model.over("preprint_date", "^269__") # item, RDM?! @out_strip def preprint_date(self, key, value): """Translates preprint_date fields.""" - date = clean_val('c', value, str) + date = clean_val("c", value, str) if date: try: date = parser.parse(date) return date.date().isoformat() except (ValueError, AttributeError): - raise ManualMigrationRequired(subfield='c') + raise ManualMigrationRequired(subfield="c") else: - raise IgnoreKey('preprint_date') + raise IgnoreKey("preprint_date") diff --git a/cds_dojson/marc21/fields/utils.py b/cds_dojson/marc21/fields/utils.py index e94d50fd..e94ee343 100644 --- a/cds_dojson/marc21/fields/utils.py +++ b/cds_dojson/marc21/fields/utils.py @@ -42,21 +42,20 @@ # TODO to be decided where is the config value for domain def related_url(value): """Builds related records urls.""" - return '{0}{1}'.format('https://cds.cern.ch/record/', value) + return "{0}{1}".format("https://cds.cern.ch/record/", value) def clean_pages_range(pages_subfield, value): """Builds pages dictionary.""" - page_regex = r'\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]*\d*)$' + page_regex = r"\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]*\d*)$" pages_val = clean_val(pages_subfield, value, str, regex_format=page_regex) if pages_val: - pages = re.split(r'[\-‐‑‒–—―⁻₋−﹘﹣－]+', pages_val) + pages = re.split(r"[\-‐‑‒–—―⁻₋−﹘﹣－]+", pages_val) if len(pages) == 1: - result = {'page_start': int(pages[0])} + result = {"page_start": int(pages[0])} return result else: - result = {'page_start': int(pages[0]), - 'page_end': int(pages[1])} + result = {"page_start": int(pages[0]), "page_end": int(pages[1])} return result @@ -78,8 +77,16 @@ def clean_str(to_clean, regex_format, req, transform=None): return cleaned -def clean_val(subfield, value, var_type, req=False, regex_format=None, - default=None, manual=False, transform=None): +def clean_val( + subfield, + value, + var_type, + req=False, + regex_format=None, + default=None, + manual=False, + transform=None, +): """ Tests values using common rules. @@ -119,8 +126,9 @@ def clean_val(subfield, value, var_type, req=False, regex_format=None, def clean_email(value): """Cleans the email field.""" if value: - email = value.strip().replace(' [CERN]', '@cern.ch'). \ - replace('[CERN]', '@cern.ch') + email = ( + value.strip().replace(" [CERN]", "@cern.ch").replace("[CERN]", "@cern.ch") + ) return email @@ -143,13 +151,22 @@ def proxy(*args, **kwargs): res = fn_decorated(*args, **kwargs) if res: if not key: - return [k.replace(phrase, replace_with).strip() - for k in res] + return [k.replace(phrase, replace_with).strip() for k in res] else: - return [dict((k, v.replace(phrase, replace_with).strip() - if k == key else v - ) - for k, v in elem.items()) for elem in res] + return [ + dict( + ( + k, + ( + v.replace(phrase, replace_with).strip() + if k == key + else v + ), + ) + for k, v in elem.items() + ) + for elem in res + ] return res return proxy @@ -165,8 +182,9 @@ def wrapper(self, key, value, **kwargs): out = f(self, key, value) if out: print(out) - clean_list = [dict((k, v) for k, v in elem.items() - if v) for elem in out if elem] + clean_list = [ + dict((k, v) for k, v in elem.items() if v) for elem in out if elem + ] clean_list = [elem for elem in clean_list if elem] if not clean_list: raise IgnoreKey(key) @@ -213,16 +231,18 @@ def _get_http_request(url, retry=0): def get_author_info_from_people_collection(info): """Get author information from CDS auto-completion endpoint.""" # TODO: probably we will need to extract this somewhere else - URL = ('https://cds.cern.ch/submit/get_authors?' - 'query={0}&relative_curdir=cdslabs%2Fvideos') - if '0' in info or not info.get('a'): + URL = ( + "https://cds.cern.ch/submit/get_authors?" + "query={0}&relative_curdir=cdslabs%2Fvideos" + ) + if "0" in info or not info.get("a"): # There is already enough information or we don't have a name to query return info - author_name = info.get('a') + author_name = info.get("a") if PY2: # In Python 3, encoded name will change type to bytes and this will # cause query to CDS to fail - author_name = author_name.encode('utf-8') + author_name = author_name.encode("utf-8") author_info = _get_http_request(url=URL.format(author_name), retry=10) if not author_info or len(author_info) > 1: # Didn't find anything or find to many matches @@ -230,98 +250,98 @@ def get_author_info_from_people_collection(info): # Prepare author name author_info = author_info[0] - if 'name' not in author_info: - author_info['name'] = '{0}, {1}'.format(author_info['lastname'], - author_info['firstname']) - return MementoDict([ - (k, v) for k, v in chain(info.items(), iteritems(author_info))]) + if "name" not in author_info: + author_info["name"] = "{0}, {1}".format( + author_info["lastname"], author_info["firstname"] + ) + return MementoDict([(k, v) for k, v in chain(info.items(), iteritems(author_info))]) def _get_correct_video_contributor_role(role): """Clean up roles for Videos.""" translations = { - '3d animation': 'Animatons by', - '3d animations': 'Animations by', - 'animation': 'Animations by', - 'animations': 'Animations by', - u'auteur-r\xe9alisateur': ('Creator', 'Producer'), - 'author': 'Creator', - 'autor': 'Creator', - 'camera': 'Camera Operator', - 'camera & sound': 'Camera Operator', - 'co-produced by': 'Co-Producer', - 'co-production': 'Co-Producer', - 'commentaire': 'Comments by', - 'commentaires': 'Comments by', - 'content': 'Screenwriter', - 'coordination du film': 'Editor', - 'coordination montage film': 'Editor', - 'copione & realizzazione': ('Screenwriter', 'Producer'), - 'created by': 'Creator', - 'credit': 'Credits', - 'credits': 'Credits', - 'diaporama': 'Photography', - 'dierctor': 'Director', - 'directed by': 'Director', - 'directeur de production': ('Director', 'Producer'), - 'direction and project management': ('Director', ''), - 'director': 'Director', - 'director & producer': ('Director', 'Producer'), - 'directors': 'Director', - 'direttore': 'Director', - 'ecriture': 'Screenwriter', - 'edited by': 'Editor', - 'editing': 'Editor', - 'edition': 'Editor', - 'editor': 'Editor', - u'enqu\xeate': 'Reporter', - 'entretiens': 'Reporter', - 'executive producer': 'Producer', - 'film maker': 'Camera Operator', - 'filmed by': 'Camera Operator', - 'filmmaker': 'Camera Operator', - 'images': 'Photography', - 'images & editing': ('Photography', 'Editor'), - 'images from': 'Photography', - 'images maker': 'Photography', - 'journalist': 'Reporter', - 'konzept und herstellung': 'Producer', - 'made by': 'Creator', - 'montage': 'Editor', - 'narrator': 'Narrator', - 'presentator': 'Reporter', - 'presented by': 'Reporter', - 'presenter': 'Reporter', - 'presenter/reporter': 'Reporter', - 'produced by': 'Producer', - 'producer': 'Producer', - 'producers': 'Producer', - 'production': 'Producer', - 'production et commentaire': ('Producer', 'Comments by'), - 'productor': 'Producer', - 'realisateur': 'Producer', - 'realisation': 'Producer', - u'r\xe9alisation': 'Producer', - 'redaktion': 'Editor', - 'related by': 'Narrator', - 'reporter': 'Reporter', - 'reports': 'Reporter', - 'scenario et realisation': 'Producer', - 'script': 'Screenwriter', - 'script & director': ('Screenwriter', 'Director'), - 'script and director': ('Screenwriter', 'Director'), - 'script, design and direction': ('Screenwriter', 'Editor', 'Director'), - u'sc\xe9nario et r\xe9alisation': 'Producer', - 'series producer': 'Producer', - 'shooting and editing': ('Camera Operator', 'Editor'), - 'son': 'Music by', - 'speaker': 'Speaker', - 'writen by': 'Screenwriter', - 'writer and director': ('Screenwriter', 'Director'), - 'written & directed by': ('Screenwriter', 'Director'), - 'written & produced by': ('Screenwriter', 'Producer'), - 'written and director': ('Screenwriter', 'Director'), - 'written and produced by': ('Screenwriter', 'Producer'), + "3d animation": "Animatons by", + "3d animations": "Animations by", + "animation": "Animations by", + "animations": "Animations by", + "auteur-r\xe9alisateur": ("Creator", "Producer"), + "author": "Creator", + "autor": "Creator", + "camera": "Camera Operator", + "camera & sound": "Camera Operator", + "co-produced by": "Co-Producer", + "co-production": "Co-Producer", + "commentaire": "Comments by", + "commentaires": "Comments by", + "content": "Screenwriter", + "coordination du film": "Editor", + "coordination montage film": "Editor", + "copione & realizzazione": ("Screenwriter", "Producer"), + "created by": "Creator", + "credit": "Credits", + "credits": "Credits", + "diaporama": "Photography", + "dierctor": "Director", + "directed by": "Director", + "directeur de production": ("Director", "Producer"), + "direction and project management": ("Director", ""), + "director": "Director", + "director & producer": ("Director", "Producer"), + "directors": "Director", + "direttore": "Director", + "ecriture": "Screenwriter", + "edited by": "Editor", + "editing": "Editor", + "edition": "Editor", + "editor": "Editor", + "enqu\xeate": "Reporter", + "entretiens": "Reporter", + "executive producer": "Producer", + "film maker": "Camera Operator", + "filmed by": "Camera Operator", + "filmmaker": "Camera Operator", + "images": "Photography", + "images & editing": ("Photography", "Editor"), + "images from": "Photography", + "images maker": "Photography", + "journalist": "Reporter", + "konzept und herstellung": "Producer", + "made by": "Creator", + "montage": "Editor", + "narrator": "Narrator", + "presentator": "Reporter", + "presented by": "Reporter", + "presenter": "Reporter", + "presenter/reporter": "Reporter", + "produced by": "Producer", + "producer": "Producer", + "producers": "Producer", + "production": "Producer", + "production et commentaire": ("Producer", "Comments by"), + "productor": "Producer", + "realisateur": "Producer", + "realisation": "Producer", + "r\xe9alisation": "Producer", + "redaktion": "Editor", + "related by": "Narrator", + "reporter": "Reporter", + "reports": "Reporter", + "scenario et realisation": "Producer", + "script": "Screenwriter", + "script & director": ("Screenwriter", "Director"), + "script and director": ("Screenwriter", "Director"), + "script, design and direction": ("Screenwriter", "Editor", "Director"), + "sc\xe9nario et r\xe9alisation": "Producer", + "series producer": "Producer", + "shooting and editing": ("Camera Operator", "Editor"), + "son": "Music by", + "speaker": "Speaker", + "writen by": "Screenwriter", + "writer and director": ("Screenwriter", "Director"), + "written & directed by": ("Screenwriter", "Director"), + "written & produced by": ("Screenwriter", "Producer"), + "written and director": ("Screenwriter", "Director"), + "written and produced by": ("Screenwriter", "Producer"), } return translations[role.lower()] @@ -329,78 +349,70 @@ def _get_correct_video_contributor_role(role): def _get_correct_books_contributor_role(subfield, role): """Clean up roles.""" translations = { - 'author': 'AUTHOR', - 'dir.': 'SUPERVISOR', - 'dir': 'SUPERVISOR', - 'ed.': 'EDITOR', - 'ed': 'EDITOR', - 'ill.': 'ILLUSTRATOR', - 'ill': 'ILLUSTRATOR', + "author": "AUTHOR", + "dir.": "SUPERVISOR", + "dir": "SUPERVISOR", + "ed.": "EDITOR", + "ed": "EDITOR", + "ill.": "ILLUSTRATOR", + "ill": "ILLUSTRATOR", } if isinstance(role, str): clean_role = role.lower() else: - raise UnexpectedValue(subfield=subfield, message=' unknown role') + raise UnexpectedValue(subfield=subfield, message=" unknown role") if clean_role not in translations: - raise UnexpectedValue(subfield=subfield, message=' unknown role') + raise UnexpectedValue(subfield=subfield, message=" unknown role") return translations[clean_role] -def _extract_json_ids(info, provenence='source'): +def _extract_json_ids(info, provenence="source"): """Extract author IDs from MARC tags.""" SOURCES = { - 'AUTHOR|(INSPIRE)': 'INSPIRE ID', - 'AUTHOR|(CDS)': 'CDS', - 'AUTHOR|(SzGeCERN)': 'CERN' + "AUTHOR|(INSPIRE)": "INSPIRE ID", + "AUTHOR|(CDS)": "CDS", + "AUTHOR|(SzGeCERN)": "CERN", } - regex = re.compile(r'((AUTHOR\|\((CDS|INSPIRE)\))|(\(SzGeCERN\)))(.*)') + regex = re.compile(r"((AUTHOR\|\((CDS|INSPIRE)\))|(\(SzGeCERN\)))(.*)") ids = [] - author_ids = force_list(info.get('0', '')) + author_ids = force_list(info.get("0", "")) for author_id in author_ids: match = regex.match(author_id) if match: - ids.append( - { - 'value': match.group(3), - provenence: SOURCES[match.group(1)] - }) + ids.append({"value": match.group(3), provenence: SOURCES[match.group(1)]}) # Try and get the IDs from the auto-completion try: - ids.append({'value': info['cernccid'], provenence: 'CERN'}) + ids.append({"value": info["cernccid"], provenence: "CERN"}) except KeyError: pass try: - ids.append({'value': info['recid'], provenence: 'CDS'}) + ids.append({"value": info["recid"], provenence: "CDS"}) except KeyError: pass try: - ids.append({'value': info['inspireid'], provenence: 'INSPIRE ID'}) + ids.append({"value": info["inspireid"], provenence: "INSPIRE ID"}) except KeyError: pass return ids -def _extract_json_ils_ids(info, provenence='source'): +def _extract_json_ils_ids(info, provenence="source"): """Extract author IDs from MARC tags.""" SOURCES = { - 'AUTHOR|(INSPIRE)': 'INSPIRE ID', - 'AUTHOR|(CDS)': 'CDS', - 'AUTHOR|(SzGeCERN)': 'CERN' + "AUTHOR|(INSPIRE)": "INSPIRE ID", + "AUTHOR|(CDS)": "CDS", + "AUTHOR|(SzGeCERN)": "CERN", } - regex = re.compile(r'(AUTHOR\|\((CDS|INSPIRE|SzGeCERN)\))(.*)') + regex = re.compile(r"(AUTHOR\|\((CDS|INSPIRE|SzGeCERN)\))(.*)") ids = [] - author_ids = force_list(info.get('0', '')) + author_ids = force_list(info.get("0", "")) for author_id in author_ids: match = regex.match(author_id) if match: - ids.append( - { - 'value': match.group(3), - provenence: SOURCES[match.group(1)] - }) + ids.append({"value": match.group(3), provenence: SOURCES[match.group(1)]}) try: - ids.append({'value': info['inspireid'], provenence: 'INSPIRE ID'}) + ids.append({"value": info["inspireid"], provenence: "INSPIRE ID"}) except KeyError: pass @@ -409,30 +421,24 @@ def _extract_json_ils_ids(info, provenence='source'): def build_contributor_books(value): """Create the contributors for books.""" - if not value.get('a'): + if not value.get("a"): return [] contributor = { - 'identifiers': _extract_json_ils_ids(value, 'scheme') or None, - 'full_name': value.get('name') or clean_val('a', value, str), - 'roles': [ - _get_correct_books_contributor_role( - 'e', value.get('e', 'author')) - ], + "identifiers": _extract_json_ils_ids(value, "scheme") or None, + "full_name": value.get("name") or clean_val("a", value, str), + "roles": [_get_correct_books_contributor_role("e", value.get("e", "author"))], } - value_u = value.get('u') + value_u = value.get("u") if value_u: values_u_list = list(force_list(value_u)) - other = ['et al.', 'et al'] + other = ["et al.", "et al"] for x in other: if x in values_u_list: values_u_list.remove(x) - contributor['affiliations'] = [{'name': x} for x in - values_u_list] - contributor = dict( - (k, v) for k, v in iteritems(contributor) if v is not None - ) + contributor["affiliations"] = [{"name": x} for x in values_u_list] + contributor = dict((k, v) for k, v in iteritems(contributor) if v is not None) return contributor @@ -449,72 +455,92 @@ def build_contributor_videos(value): as many times as the roles (2 in the example). """ OLD_VIDEO_TEAM_NAMES = { - 'cern', 'cern ', 'cern / audiovisual service', 'cern ???', - 'cern audio service', 'cern audio video service', - 'cern audio visual service', 'cern audioivisual unit', - 'cern audiovideo service', 'cern audiovisual', - 'cern audiovisual production', 'cern audiovisual production service', - 'cern audiovisual productions', 'cern audiovisual productions service', - 'cern audiovisual servic', 'cern audiovisual service', - 'cern audiovisual team', 'cern audiovisual unit', - 'cern audiovisual unit ', 'cern audiovisual unit ??', - 'cern audivisual service', 'cern av service', 'cern avc', - 'cern multimedia', 'cern multimedia production unit', - 'cern multimedia productions', 'cern production', - 'cern video procuctions', 'cern video production', - 'cern video productions', 'cern video service', - 'cern video+photo productions', 'cern videos production', - 'cern videos productions', 'cern visual media office', 'cern vmo' + "cern", + "cern ", + "cern / audiovisual service", + "cern ???", + "cern audio service", + "cern audio video service", + "cern audio visual service", + "cern audioivisual unit", + "cern audiovideo service", + "cern audiovisual", + "cern audiovisual production", + "cern audiovisual production service", + "cern audiovisual productions", + "cern audiovisual productions service", + "cern audiovisual servic", + "cern audiovisual service", + "cern audiovisual team", + "cern audiovisual unit", + "cern audiovisual unit ", + "cern audiovisual unit ??", + "cern audivisual service", + "cern av service", + "cern avc", + "cern multimedia", + "cern multimedia production unit", + "cern multimedia productions", + "cern production", + "cern video procuctions", + "cern video production", + "cern video productions", + "cern video service", + "cern video+photo productions", + "cern videos production", + "cern videos productions", + "cern visual media office", + "cern vmo", } - if not value.get('a'): + if not value.get("a"): # Sometimes there are 100 or 700 fields with no $a subfield # (wrong metadata), so let's ignore them - value.get('e') # quick hack to avoid false positves + value.get("e") # quick hack to avoid false positves return [] - if value.get('a').lower() not in OLD_VIDEO_TEAM_NAMES: + if value.get("a").lower() not in OLD_VIDEO_TEAM_NAMES: # Avoids a few calls value = get_author_info_from_people_collection(value) role = _get_correct_video_contributor_role( - value.get('e', 'producer')) # always unicode + value.get("e", "producer") + ) # always unicode contributors = [] contributor = { - 'ids': _extract_json_ids(value) or None, - 'name': value.get('name') or value.get('a'), - 'affiliations': force_list(value.get('affiliation') or value.get('u')), - 'email': value.get('email'), + "ids": _extract_json_ids(value) or None, + "name": value.get("name") or value.get("a"), + "affiliations": force_list(value.get("affiliation") or value.get("u")), + "email": value.get("email"), } - if contributor['name'].lower() in OLD_VIDEO_TEAM_NAMES: - contributor['name'] = 'CERN Video Productions' + if contributor["name"].lower() in OLD_VIDEO_TEAM_NAMES: + contributor["name"] = "CERN Video Productions" - contributor = dict( - (k, v) for k, v in iteritems(contributor) if v is not None - ) + contributor = dict((k, v) for k, v in iteritems(contributor) if v is not None) if isinstance(role, tuple): for _role in role: - contributor['role'] = _role + contributor["role"] = _role contributors.append(copy.deepcopy(contributor)) else: - contributor['role'] = role + contributor["role"] = role contributors.append(contributor) return contributors def build_contributor_from_508(value): """Build contributors from field 508.""" - item = value.get('a') - if item.lower().startswith('camera operator'): - if ',' in item: - camera_operators = value.get('a').split(',') + item = value.get("a") + if item.lower().startswith("camera operator"): + if "," in item: + camera_operators = value.get("a").split(",") # remove "camera operator" from the list camera_operators.pop(0) contributors = [] for name in camera_operators: contributor = build_contributor_videos( - {'a': name.strip(), 'e': 'camera'}) + {"a": name.strip(), "e": "camera"} + ) if contributor: contributors.append(contributor[0]) return contributors else: - return build_contributor_videos({'a': item.strip(), 'e': 'credits'}) + return build_contributor_videos({"a": item.strip(), "e": "credits"}) diff --git a/cds_dojson/marc21/fields/videos/base.py b/cds_dojson/marc21/fields/videos/base.py index 00a2550d..944be238 100644 --- a/cds_dojson/marc21/fields/videos/base.py +++ b/cds_dojson/marc21/fields/videos/base.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Common videos fields.""" + from __future__ import absolute_import, print_function from dojson.errors import IgnoreKey @@ -30,66 +31,64 @@ from ...models.videos.base import model -@model.over('report_number', '^(037|088)__') +@model.over("report_number", "^(037|088)__") @for_each_value def report_number(self, key, value): """Report number. Category and type are also derived from the report number. """ - rn = value.get('a') or value.get('9') - if rn and key.startswith('037__'): + rn = value.get("a") or value.get("9") + if rn and key.startswith("037__"): # Extract category and type only from main report number, i.e. 037__a - self['category'], self['type'] = rn.split('-')[:2] + self["category"], self["type"] = rn.split("-")[:2] return rn -@model.over('contributors', '^(100|700|508)__') +@model.over("contributors", "^(100|700|508)__") def contributors(self, key, value): """Contributors.""" - authors = self.get('contributors', []) - if key in ['100__', '700__']: + authors = self.get("contributors", []) + if key in ["100__", "700__"]: items = build_contributor_videos(value) else: items = build_contributor_from_508(value) # add only contributors that are not part of the authors if items: - authors.extend( - [item for item in items if item and item not in authors] - ) + authors.extend([item for item in items if item and item not in authors]) return authors -@model.over('title', '^245_[1_]') +@model.over("title", "^245_[1_]") @filter_values def title(self, key, value): """Title.""" return { - 'title': value.get('a'), - 'subtitle': value.get('b'), + "title": value.get("a"), + "subtitle": value.get("b"), } -@model.over('description', '^520__') +@model.over("description", "^520__") def description(self, key, value): """Description.""" - return value.get('a') + return value.get("a") -@model.over('license', '^540__') +@model.over("license", "^540__") @for_each_value @filter_values def license(self, key, value): """License.""" return { - 'license': value.get('a'), - 'material': value.get('3'), - 'url': value.get('u'), + "license": value.get("a"), + "material": value.get("3"), + "url": value.get("u"), } -@model.over('_access', '(^859__)|(^506[1_]_)') +@model.over("_access", "(^859__)|(^506[1_]_)") def access(self, key, value): """Access rights. @@ -99,62 +98,65 @@ def access(self, key, value): record. The groups are in the form [CERN] which needs to be transform into the email form. """ - _access = self.get('_access', {}) - if key == '859__' and 'f' in value: - _access.setdefault('update', []) - _access['update'].append(value.get('f')) - elif key.startswith('506'): - _access.setdefault('read', []) - _access['read'].extend([ - s.replace(' [CERN]', '@cern.ch') - for s in force_list(value.get('d') or value.get('m', '')) if s - ]) + _access = self.get("_access", {}) + if key == "859__" and "f" in value: + _access.setdefault("update", []) + _access["update"].append(value.get("f")) + elif key.startswith("506"): + _access.setdefault("read", []) + _access["read"].extend( + [ + s.replace(" [CERN]", "@cern.ch") + for s in force_list(value.get("d") or value.get("m", "")) + if s + ] + ) return _access -@model.over('external_system_identifiers', '^970__') +@model.over("external_system_identifiers", "^970__") @for_each_value def external_system_identifiers(self, key, value): """External unique identifiers.""" - value = value.get('a', '') + value = value.get("a", "") return { - 'value': value, - "schema": 'ALEPH' if value.startswith('0000') else value[:3] + "value": value, + "schema": "ALEPH" if value.startswith("0000") else value[:3], } -@model.over('note', '^(5904_|500__)') +@model.over("note", "^(5904_|500__)") def note(self, key, value): """Note.""" - return value.get('a') + return value.get("a") -@model.over('translations', '(^246_[1_])|(590__)') +@model.over("translations", "(^246_[1_])|(590__)") @ignore_value def translations(self, key, value): """Translations.""" - translation = self.get('translations', [{}])[0] - if key.startswith('246'): - translation['title'] = {'title': value.get('a')} - if key.startswith('590'): - translation['description'] = value.get('a') - translation['language'] = 'fr' - self['translations'] = [translation] - raise IgnoreKey('translations') + translation = self.get("translations", [{}])[0] + if key.startswith("246"): + translation["title"] = {"title": value.get("a")} + if key.startswith("590"): + translation["description"] = value.get("a") + translation["language"] = "fr" + self["translations"] = [translation] + raise IgnoreKey("translations") -@model.over('original_source', '^541__') +@model.over("original_source", "^541__") def original_source(self, key, value): """Original source.""" - return value.get('e') + return value.get("e") -@model.over('keywords', '^6531_') +@model.over("keywords", "^6531_") @for_each_value @filter_values def keywords(self, key, value): """Keywords.""" return { - 'name': clean_val('a', value, str), - 'source': value.get('9') or value.get('g'), # Easier to solve here + "name": clean_val("a", value, str), + "source": value.get("9") or value.get("g"), # Easier to solve here } diff --git a/cds_dojson/marc21/fields/videos/project.py b/cds_dojson/marc21/fields/videos/project.py index ca29da77..c05e5309 100644 --- a/cds_dojson/marc21/fields/videos/project.py +++ b/cds_dojson/marc21/fields/videos/project.py @@ -26,12 +26,12 @@ from ...models.videos.project import model -@model.over('related_links', '^773__') +@model.over("related_links", "^773__") @for_each_value @filter_values def related_links(self, key, value): """Related links.""" return { - 'name': value.get('p'), - 'url': value.get('u'), + "name": value.get("p"), + "url": value.get("u"), } diff --git a/cds_dojson/marc21/fields/videos/utils.py b/cds_dojson/marc21/fields/videos/utils.py index d6455cfa..8968e811 100644 --- a/cds_dojson/marc21/fields/videos/utils.py +++ b/cds_dojson/marc21/fields/videos/utils.py @@ -28,17 +28,17 @@ def language_to_isocode(lang): return pycountry.languages.get(alpha_3=lang).alpha_2 except (KeyError, AttributeError): exceptions = { - 'chi': 'zh', - 'cze': 'cs', - 'dut': 'nl', - 'eng-fre': 'en-fr', - 'fre': 'fr', - 'ger': 'de', - 'gre': 'el', - 'sil': 'silent', - 'silent': 'silent', - 'sl': 'sl', - 'sr': 'sr', + "chi": "zh", + "cze": "cs", + "dut": "nl", + "eng-fre": "en-fr", + "fre": "fr", + "ger": "de", + "gre": "el", + "sil": "silent", + "silent": "silent", + "sl": "sl", + "sr": "sr", } value = exceptions.get(lang) if value: @@ -47,4 +47,4 @@ def language_to_isocode(lang): return pycountry.languages.get(alpha_2=lang[0:2]).alpha_2 except (KeyError, AttributeError): # FIXME log somewhere - return 'silent' + return "silent" diff --git a/cds_dojson/marc21/fields/videos/video.py b/cds_dojson/marc21/fields/videos/video.py index 192b58f3..1bdc454d 100644 --- a/cds_dojson/marc21/fields/videos/video.py +++ b/cds_dojson/marc21/fields/videos/video.py @@ -22,6 +22,7 @@ import os import re + try: from collections import defaultdict except ImportError: @@ -40,7 +41,7 @@ # Required fields -@model.over('duration', '^300__') +@model.over("duration", "^300__") def duration(self, key, value): """Duration. @@ -49,275 +50,280 @@ def duration(self, key, value): off dealing with more regex. """ try: - return re.match(r'(\d{2}:\d{2}:\d{2})(\.\d+)?', value.get('a')) \ - .group(1) + return re.match(r"(\d{2}:\d{2}:\d{2})(\.\d+)?", value.get("a")).group(1) except (AttributeError, TypeError): # The regex didn't match, we will extract the duration later. - return '00:00:00' + return "00:00:00" # Language -@model.over('language', '^041__') + +@model.over("language", "^041__") def language(self, key, value): """Language.""" - return language_to_isocode(value.get('a')) + return language_to_isocode(value.get("a")) # Rest -@model.over('physical_medium', '(^340__)|(^852__)') + +@model.over("physical_medium", "(^340__)|(^852__)") def physical_medium(self, key, value): """Physical medium.""" + def find_match(seq, copy): - if not seq and not copy \ - and key == '852__' and len(_physical_medium) == 1: + if not seq and not copy and key == "852__" and len(_physical_medium) == 1: return _physical_medium[0] else: for i in _physical_medium: - if seq and seq in i.get('sequence_number') \ - or copy and copy == i.get('copy_number'): + if ( + seq + and seq in i.get("sequence_number") + or copy + and copy == i.get("copy_number") + ): return i _physical_medium.append({}) return _physical_medium[-1] - _physical_medium = self.get('physical_medium', []) + _physical_medium = self.get("physical_medium", []) sequence_numbers = [] for value in force_list(value): - pm = find_match(value.get('8'), value.get('y')) + pm = find_match(value.get("8"), value.get("y")) # Append ``_8`` and ``_9`` - sequence_numbers.append(value.get('8')) - sequence_numbers.append(value.get('9')) - if key == '340__': - pm.update({ - 'medium_standard': value.get('a'), - 'note': value.get('j'), - 'camera': value.get('d'), - 'arrangement': value.get('k'), - 'copy_number': value.get('t'), - }) - elif key == '852__': - pm.update({ - 'internal_note': value.get('9'), - 'location': value.get('a'), - 'shelf': value.get('b'), - 'bar_code': value.get('c'), - 'copy_number': value.get('t'), - 'note': value.get('z'), - }) - pm.update({ - 'sequence_number': [ - sequence_number for sequence_number in set(sequence_numbers) - if sequence_number is not None - ] - }) - return [dict((k, v) for k, v in iteritems(i) if v is not None) - for i in _physical_medium] - - -@model.over('_project_id', '^773__') + sequence_numbers.append(value.get("8")) + sequence_numbers.append(value.get("9")) + if key == "340__": + pm.update( + { + "medium_standard": value.get("a"), + "note": value.get("j"), + "camera": value.get("d"), + "arrangement": value.get("k"), + "copy_number": value.get("t"), + } + ) + elif key == "852__": + pm.update( + { + "internal_note": value.get("9"), + "location": value.get("a"), + "shelf": value.get("b"), + "bar_code": value.get("c"), + "copy_number": value.get("t"), + "note": value.get("z"), + } + ) + pm.update( + { + "sequence_number": [ + sequence_number + for sequence_number in set(sequence_numbers) + if sequence_number is not None + ] + } + ) + return [ + dict((k, v) for k, v in iteritems(i) if v is not None) for i in _physical_medium + ] + + +@model.over("_project_id", "^773__") @ignore_value def project_id(self, key, value): """Report number.""" values = force_list(value) project_id = None - related_links = self.get('related_links', []) + related_links = self.get("related_links", []) for value in values: related_link = {} - if 'p' in value and 'u' in value: - related_link['name'] = value.get('p') - related_link['url'] = value.get('u') + if "p" in value and "u" in value: + related_link["name"] = value.get("p") + related_link["url"] = value.get("u") related_links.append(related_link) else: - project_id = value.get('u') + project_id = value.get("u") if related_links: - self['related_links'] = related_links + self["related_links"] = related_links if not project_id: - raise IgnoreKey('project_id') + raise IgnoreKey("project_id") return project_id -@model.over('location', '^110__') +@model.over("location", "^110__") def location(self, key, value): """Location.""" - return value.get('a') + return value.get("a") -@model.over('internal_note', '^595__') +@model.over("internal_note", "^595__") @ignore_value def internal_note(self, key, value): """Internal note.""" - CATEGS = ('CERN50', 'CERN EDS', 'Video-SR-F', 'Pilote PICTURAE', 'Press') + CATEGS = ("CERN50", "CERN EDS", "Video-SR-F", "Pilote PICTURAE", "Press") _internal_categories = defaultdict(list) - _internal_categories.update(self.get('internal_categories', {})) - _internal_notes = self.get('internal_note', '').splitlines() + _internal_categories.update(self.get("internal_categories", {})) + _internal_notes = self.get("internal_note", "").splitlines() for v in force_list(value): - if v.get('a') in CATEGS: - _internal_categories[v.get('a')].append(v.get('s')) + if v.get("a") in CATEGS: + _internal_categories[v.get("a")].append(v.get("s")) else: - _internal_notes.append(v.get('a')) + _internal_notes.append(v.get("a")) if _internal_categories: - self['internal_categories'] = dict(_internal_categories) - return '\n'.join(_internal_notes) or None + self["internal_categories"] = dict(_internal_categories) + return "\n".join(_internal_notes) or None -@model.over('subject', '^65017') +@model.over("subject", "^65017") def subject(self, key, value): """Subject.""" - return { - 'source': value.get('2', 'SzGeCERN'), - 'term': value.get('a') - } + return {"source": value.get("2", "SzGeCERN"), "term": value.get("a")} -@model.over('accelerator_experiment', '^693__') +@model.over("accelerator_experiment", "^693__") @filter_values def accelerator_experiment(self, key, value): """Accelerator experiments.""" return { - 'accelerator': value.get('a'), - 'experiment': value.get('e'), - 'study': value.get('s'), - 'facility': value.get('f'), - 'project': value.get('p'), + "accelerator": value.get("a"), + "experiment": value.get("e"), + "study": value.get("s"), + "facility": value.get("f"), + "project": value.get("p"), } -@model.over('date', '^269__') +@model.over("date", "^269__") def date(self, key, value): """Date.""" - return arrow.get(value.get('c')).strftime('%Y-%m-%d') + return arrow.get(value.get("c")).strftime("%Y-%m-%d") -@model.over('copyright', '^542__') +@model.over("copyright", "^542__") @filter_values def copyright(self, key, value): """Copyright.""" return { - 'holder': value.get('d'), - 'year': value.get('g'), - 'message': value.get('f'), + "holder": value.get("d"), + "year": value.get("g"), + "message": value.get("f"), } -@model.over('_files', '^8567_') +@model.over("_files", "^8567_") @for_each_value @filter_values def _files(self, key, value): """File list.""" + def get_key(value): - if value.get('d'): - return value.get('d').split('\\')[-1] + if value.get("d"): + return value.get("d").split("\\")[-1] else: - return os.path.basename(value.get('u')) + return os.path.basename(value.get("u")) def get_context_type(value): - if value.get('d'): - return 'master', 'video' - if value.get('y').startswith('thumbnail'): - return 'ignore', 'ignore' - if value.get('x').startswith('wmv'): - return 'ignore', 'ignore' - if value.get('x').startswith('flv'): - return 'ignore', 'ignore' - if 'kbps maxH' in value.get('y'): - return 'subformat', 'video' - if value.get('x') == 'subtitle': - return 'playlist', 'text' - if value.get('y').startswith('posterframe'): - if ' 5 percent' in value.get('y'): - return 'poster', 'image' + if value.get("d"): + return "master", "video" + if value.get("y").startswith("thumbnail"): + return "ignore", "ignore" + if value.get("x").startswith("wmv"): + return "ignore", "ignore" + if value.get("x").startswith("flv"): + return "ignore", "ignore" + if "kbps maxH" in value.get("y"): + return "subformat", "video" + if value.get("x") == "subtitle": + return "playlist", "text" + if value.get("y").startswith("posterframe"): + if " 5 percent" in value.get("y"): + return "poster", "image" else: - return 'frame', 'image' + return "frame", "image" return None, None def get_tags(context_type, value): - if context_type == 'poster': - wh = value.get('y').split(' ')[1] - [width, height] = wh.split('x') - return {'width': width, 'height': height} - if value.get('x') == 'subtitle': - return { - 'language': language_to_isocode( - value.get('y').split(' ')[1][:3]) - } - if context_type == 'master': - return {'preview': True} + if context_type == "poster": + wh = value.get("y").split(" ")[1] + [width, height] = wh.split("x") + return {"width": width, "height": height} + if value.get("x") == "subtitle": + return {"language": language_to_isocode(value.get("y").split(" ")[1][:3])} + if context_type == "master": + return {"preview": True} return {} def get_filepath(value): - if value.get('d'): - return value.get('d')[ - len('\\\\cern.ch\\dfs\\Services\\'): - ].replace('\\', '/') - else: - return re.sub( - 'https?://mediaarchive.cern.ch/', '', value.get('u', '') + if value.get("d"): + return value.get("d")[len("\\\\cern.ch\\dfs\\Services\\") :].replace( + "\\", "/" ) + else: + return re.sub("https?://mediaarchive.cern.ch/", "", value.get("u", "")) def get_tags_to_guess_preset(context_type, value): - info = value.get('y').split(' ') - return { - 'video_bitrate': int(info[0]), - 'preset': '{0}p'.format(info[3]) - } + info = value.get("y").split(" ") + return {"video_bitrate": int(info[0]), "preset": "{0}p".format(info[3])} def get_tags_to_transform(context_type, value): - if context_type in ['frame', 'poster']: - return {'timestamp': int(value.get('y').split(' ')[3])} + if context_type in ["frame", "poster"]: + return {"timestamp": int(value.get("y").split(" ")[3])} def get_frame_name(result): - _, ext = os.path.splitext(result['key']) - index = (int(result['tags_to_transform']['timestamp']) // 10) + 1 + _, ext = os.path.splitext(result["key"]) + index = (int(result["tags_to_transform"]["timestamp"]) // 10) + 1 return "frame-{0}{1}".format(index, ext) # ignore 'x' sometimes (when is not useful) - value.get('x') + value.get("x") def compute(value, context_type, media_type): result = {} - result['key'] = get_key(value) - result['tags'] = get_tags(context_type, value) - result['tags'].update(context_type=context_type, media_type=media_type) - result['tags']['content_type'] = os.path.splitext(result['key'])[1][1:] - result['filepath'] = get_filepath(value) - if context_type == 'subformat': - result['tags_to_guess_preset'] = get_tags_to_guess_preset( - context_type, value) - result['tags_to_transform'] = get_tags_to_transform( - context_type, value) - - if result['key'].startswith('proxy-') or context_type == 'ignore': + result["key"] = get_key(value) + result["tags"] = get_tags(context_type, value) + result["tags"].update(context_type=context_type, media_type=media_type) + result["tags"]["content_type"] = os.path.splitext(result["key"])[1][1:] + result["filepath"] = get_filepath(value) + if context_type == "subformat": + result["tags_to_guess_preset"] = get_tags_to_guess_preset( + context_type, value + ) + result["tags_to_transform"] = get_tags_to_transform(context_type, value) + + if result["key"].startswith("proxy-") or context_type == "ignore": # skip proxy files - raise IgnoreKey('_files') + raise IgnoreKey("_files") - if context_type == 'frame': + if context_type == "frame": # update key name - result['key'] = get_frame_name(result) + result["key"] = get_frame_name(result) return result result = compute(deepcopy(value), *get_context_type(value)) # if it's the poster frame, make a copy for a frame! - if result['tags']['context_type'] == 'poster' and \ - result['tags_to_transform']['timestamp'] == 5: - frame_5 = compute(value, 'frame', 'image') - if '_files' not in self: - self['_files'] = [] - self['_files'].append(frame_5) + if ( + result["tags"]["context_type"] == "poster" + and result["tags_to_transform"]["timestamp"] == 5 + ): + frame_5 = compute(value, "frame", "image") + if "_files" not in self: + self["_files"] = [] + self["_files"].append(frame_5) # update posterframe key name - _, ext = os.path.splitext(result['key']) - result['key'] = 'posterframe{0}'.format(ext) + _, ext = os.path.splitext(result["key"]) + result["key"] = "posterframe{0}".format(ext) return result -@model.over('audio_characteristics', '^344__') +@model.over("audio_characteristics", "^344__") def audio_characteristics(self, key, value): """Audio characteristics.""" return { - 'playback_channels': value.get('g'), + "playback_channels": value.get("g"), } diff --git a/cds_dojson/marc21/models/base.py b/cds_dojson/marc21/models/base.py index b0caee21..9e1cfe72 100644 --- a/cds_dojson/marc21/models/base.py +++ b/cds_dojson/marc21/models/base.py @@ -25,4 +25,4 @@ class Base(OverdoJSONSchema): """Base model conversion MARC21 to JSON.""" -model = Base(entry_point_group='cds_dojson.marc21.base') +model = Base(entry_point_group="cds_dojson.marc21.base") diff --git a/cds_dojson/marc21/models/books/base.py b/cds_dojson/marc21/models/books/base.py index a9496ecf..e9302fe5 100644 --- a/cds_dojson/marc21/models/books/base.py +++ b/cds_dojson/marc21/models/books/base.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Base models for common fields.""" + from __future__ import unicode_literals from copy import deepcopy @@ -32,7 +33,7 @@ def get_migration_dict(): """Return migration extra data.""" __migration_dict__ = dict( - record_type='document', + record_type="document", volumes=[], serials=[], has_serial=False, @@ -41,7 +42,7 @@ def get_migration_dict(): has_related=False, has_journal=False, tags=[], - journal_record_legacy_id='', + journal_record_legacy_id="", eitems_proxy=[], eitems_has_proxy=False, eitems_file_links=[], @@ -55,105 +56,105 @@ def get_migration_dict(): return deepcopy(__migration_dict__) + COMMON_IGNORE_FIELDS = { - '003', - '005', - '020__q', - '020__c', - '020__C', - '0248_a', - '0248_p', - '041__h', # 206 cds-dojson - '035__z', - '037__c', # arXiv subject category - '050__b', - '050_4b', - '082002', - '082042', - '0820_2', - '082__2', # INSPIRE keywords - '084__a', # INSPIRE keywords - '084__2', - '100__9', - '111__d', - '111__f', - '145__a', - '246__i', - '269__a', # preprint info - '269__b', # preprint info - '269__c', # preprint date - '270__m', # conference email - '300__b', # 206 cds-dojson - '340__a', - '440_3a', # 206 cds-dojson - '541__9', - '541__a', - '541__h', - '502__a', # thesis_info/defense_date - '502__b', # thesis_info/degree_type - '502__c', # thesis_info/institutions - '502__d', # thesis_info/date (publication) - '5208_a', # 206 cds-dojson - '520__9', - '536__a', # founding info, dropped - '536__c', - '536__f', - '536__r', - '540__b', - '540__f', - '595__z', - '595__9', - '650172', - '65017a', - '650272', - '65027a', - '690__c', # 206 cds-dojson - '694__9', - '694__a', - '695__2', - '695__a', - '700__9', - '710__5', - '773__r', # publication_info/parent_report_number - '773__w', # inspire cnum (duplicated field with 035__9) - '773__z', # publication_info/parent_isbn - '775__c', # related edition's year (it will be resolved) - '852__c', - '852__h', - '852__p', - '8564_8', # bibdoc id - '8564_s', # file identifier - '8564_x', # subformat identifier - '900__s', # 206 cds-dojson - '900__u', # 206 cds-dojson - '900__y', # 206 cds-dojson - '901__a', # record affiliation - '901__u', - '916__a', - '916__d', - '916__e', - '916__y', - '940__u', - '961__c', - '961__h', - '961__l', - '961__x', - '962__b', - '962__n', # books connected by conference - '963__a', - '964__a', - '970__a', - '970__d', - '980__c', - '981__a', + "003", + "005", + "020__q", + "020__c", + "020__C", + "0248_a", + "0248_p", + "041__h", # 206 cds-dojson + "035__z", + "037__c", # arXiv subject category + "050__b", + "050_4b", + "082002", + "082042", + "0820_2", + "082__2", # INSPIRE keywords + "084__a", # INSPIRE keywords + "084__2", + "100__9", + "111__d", + "111__f", + "145__a", + "246__i", + "269__a", # preprint info + "269__b", # preprint info + "269__c", # preprint date + "270__m", # conference email + "300__b", # 206 cds-dojson + "340__a", + "440_3a", # 206 cds-dojson + "541__9", + "541__a", + "541__h", + "502__a", # thesis_info/defense_date + "502__b", # thesis_info/degree_type + "502__c", # thesis_info/institutions + "502__d", # thesis_info/date (publication) + "5208_a", # 206 cds-dojson + "520__9", + "536__a", # founding info, dropped + "536__c", + "536__f", + "536__r", + "540__b", + "540__f", + "595__z", + "595__9", + "650172", + "65017a", + "650272", + "65027a", + "690__c", # 206 cds-dojson + "694__9", + "694__a", + "695__2", + "695__a", + "700__9", + "710__5", + "773__r", # publication_info/parent_report_number + "773__w", # inspire cnum (duplicated field with 035__9) + "773__z", # publication_info/parent_isbn + "775__c", # related edition's year (it will be resolved) + "852__c", + "852__h", + "852__p", + "8564_8", # bibdoc id + "8564_s", # file identifier + "8564_x", # subformat identifier + "900__s", # 206 cds-dojson + "900__u", # 206 cds-dojson + "900__y", # 206 cds-dojson + "901__a", # record affiliation + "901__u", + "916__a", + "916__d", + "916__e", + "916__y", + "940__u", + "961__c", + "961__h", + "961__l", + "961__x", + "962__b", + "962__n", # books connected by conference + "963__a", + "964__a", + "970__a", + "970__d", + "980__c", + "981__a", } class CDSOverdoBookBase(OverdoJSONSchema): """Translation base Index for CDS Books.""" - def do(self, blob, ignore_missing=True, exception_handlers=None, - init_fields=None): + def do(self, blob, ignore_missing=True, exception_handlers=None, init_fields=None): """Translate blob values and instantiate new model instance. Raises ``MissingRule`` when no rule matched and ``ignore_missing`` @@ -182,7 +183,7 @@ def do(self, blob, ignore_missing=True, exception_handlers=None, handlers.update(exception_handlers or {}) def clean_missing(exc, output, key, value): - order = output.get('__order__') + order = output.get("__order__") if order: order.remove(key) @@ -210,7 +211,7 @@ def clean_missing(exc, output, key, value): name, creator = result data = creator(output, key, value) - if getattr(creator, '__extend__', False): + if getattr(creator, "__extend__", False): existing = output.get(name, []) existing.extend(data) output[name] = existing @@ -229,5 +230,5 @@ def clean_missing(exc, output, key, value): class BooksBase(OverdoJSONSchema): """Base model conversion MARC21 to JSON.""" -model = BooksBase(bases=(), - entry_point_group='cds_dojson.marc21.books') + +model = BooksBase(bases=(), entry_point_group="cds_dojson.marc21.books") diff --git a/cds_dojson/marc21/models/books/book.py b/cds_dojson/marc21/models/books/book.py index ebd5276b..ff8ccca2 100644 --- a/cds_dojson/marc21/models/books/book.py +++ b/cds_dojson/marc21/models/books/book.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Book model.""" + from __future__ import unicode_literals from copy import deepcopy @@ -29,21 +30,23 @@ class CDSBook(CDSOverdoBookBase): """Translation Index for CDS Books.""" - __query__ = '690C_:BOOK OR 690C_:"YELLOW REPORT" OR ' \ - '980__:PROCEEDINGS OR 980__:PERI OR ' \ - '(-980:STANDARD 980:BOOK) OR ' \ - '697C_:LEGSERLIB ' \ - '-980__:DELETED -980__:MIGRATED -980:__STANDARD -596:MULTIVOLUMES' + __query__ = ( + '690C_:BOOK OR 690C_:"YELLOW REPORT" OR ' + "980__:PROCEEDINGS OR 980__:PERI OR " + "(-980:STANDARD 980:BOOK) OR " + "697C_:LEGSERLIB " + "-980__:DELETED -980__:MIGRATED -980:__STANDARD -596:MULTIVOLUMES" + ) - __schema__ = 'https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json' + __schema__ = "https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json" __model_ignore_keys__ = { - '020__b', # this field is used to match multipart monograph items as volumes + "020__b", # this field is used to match multipart monograph items as volumes } __ignore_keys__ = COMMON_IGNORE_FIELDS | __model_ignore_keys__ - __json_init_dict__ = {'_migration': {**get_migration_dict()}} + __json_init_dict__ = {"_migration": {**get_migration_dict()}} def do(self, blob, ignore_missing=True, exception_handlers=None): """Set schema after translation depending on the model.""" @@ -51,12 +54,10 @@ def do(self, blob, ignore_missing=True, exception_handlers=None): blob=blob, ignore_missing=ignore_missing, exception_handlers=exception_handlers, - init_fields=deepcopy(self.__json_init_dict__) + init_fields=deepcopy(self.__json_init_dict__), ) - json['$schema'] = self.__class__.__schema__ + json["$schema"] = self.__class__.__schema__ return json -model = CDSBook( - bases=(books_base, ), - entry_point_group='cds_dojson.marc21.book') +model = CDSBook(bases=(books_base,), entry_point_group="cds_dojson.marc21.book") diff --git a/cds_dojson/marc21/models/books/journal.py b/cds_dojson/marc21/models/books/journal.py index e688190f..fd464c7e 100644 --- a/cds_dojson/marc21/models/books/journal.py +++ b/cds_dojson/marc21/models/books/journal.py @@ -28,53 +28,53 @@ class CDSJournal(CDSOverdoBookBase): """Translation Index for CDS Books.""" - __query__ = '980__:PERI -980__:DELETED -980__:MIGRATED' + __query__ = "980__:PERI -980__:DELETED -980__:MIGRATED" - __schema__ = 'https://127.0.0.1:5000/schemas/series/series-v1.0.0.json' + __schema__ = "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json" __model_ignore_keys__ = { - '080__a', - '020__C', - '080__c', - '030__a', - '030__9', - '044__a', - '044__b', - '222__a', - '310__a', - '938__a', - '044__a', - '246_39', - '85641m', - '85641g', - '85641n', - '8564_8', # not clear, some ID from legacy - '8564_s', # timestamp of files - '8564_x', # icon uri - '6531_9', - '246_3i', - '650__a', - '690C_a', - '938__p', - '938__f', - '939__a', - '939__d', - '939__u', - '939__v', - '6531_a', - '780__i', # label of relation continues - '780__t', # title of relation continues - '785__i', # label of relation continued by - '785__t', # title of relation continued by - '85641y', - '866__g', - '866__x', - '933__a', - '962__n', - '960__a', - '960__c', - '980__a', - '980__b', + "080__a", + "020__C", + "080__c", + "030__a", + "030__9", + "044__a", + "044__b", + "222__a", + "310__a", + "938__a", + "044__a", + "246_39", + "85641m", + "85641g", + "85641n", + "8564_8", # not clear, some ID from legacy + "8564_s", # timestamp of files + "8564_x", # icon uri + "6531_9", + "246_3i", + "650__a", + "690C_a", + "938__p", + "938__f", + "939__a", + "939__d", + "939__u", + "939__v", + "6531_a", + "780__i", # label of relation continues + "780__t", # title of relation continues + "785__i", # label of relation continued by + "785__t", # title of relation continued by + "85641y", + "866__g", + "866__x", + "933__a", + "962__n", + "960__a", + "960__c", + "980__a", + "980__b", } __ignore_keys__ = COMMON_IGNORE_FIELDS | __model_ignore_keys__ @@ -84,23 +84,22 @@ def do(self, blob, ignore_missing=True, exception_handlers=None): json = super(CDSJournal, self).do( blob=blob, ignore_missing=ignore_missing, - exception_handlers=exception_handlers) - json['$schema'] = self.__class__.__schema__ - if '_migration' not in json: - json['_migration'] = {} + exception_handlers=exception_handlers, + ) + json["$schema"] = self.__class__.__schema__ + if "_migration" not in json: + json["_migration"] = {} - json['_migration'].setdefault('record_type', 'journal') - json['_migration'].setdefault('volumes', []) - json['_migration'].setdefault('is_multipart', False) - json['_migration'].setdefault('has_related', False) - json['_migration'].setdefault('items', []) - json['_migration'].setdefault('electronic_items', []) - json['_migration'].setdefault('relation_previous', None) - json['_migration'].setdefault('relation_next', None) + json["_migration"].setdefault("record_type", "journal") + json["_migration"].setdefault("volumes", []) + json["_migration"].setdefault("is_multipart", False) + json["_migration"].setdefault("has_related", False) + json["_migration"].setdefault("items", []) + json["_migration"].setdefault("electronic_items", []) + json["_migration"].setdefault("relation_previous", None) + json["_migration"].setdefault("relation_next", None) return json -model = CDSJournal( - bases=(), - entry_point_group='cds_dojson.marc21.series') +model = CDSJournal(bases=(), entry_point_group="cds_dojson.marc21.series") diff --git a/cds_dojson/marc21/models/books/multipart.py b/cds_dojson/marc21/models/books/multipart.py index f8c1e608..4740df12 100644 --- a/cds_dojson/marc21/models/books/multipart.py +++ b/cds_dojson/marc21/models/books/multipart.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Series model.""" + from __future__ import unicode_literals from copy import deepcopy @@ -29,17 +30,19 @@ class CDSMultipart(CDSOverdoBookBase): """Translation Index for CDS Books.""" - __query__ = '(596__:MULTIVOLUMES AND 690C_:BOOK OR 690C_:"YELLOW REPORT" OR ' \ - '980__:PROCEEDINGS OR ' \ - '697C_:LEGSERLIB -980__:DELETED -980__:MIGRATED)' + __query__ = ( + '(596__:MULTIVOLUMES AND 690C_:BOOK OR 690C_:"YELLOW REPORT" OR ' + "980__:PROCEEDINGS OR " + "697C_:LEGSERLIB -980__:DELETED -980__:MIGRATED)" + ) - __schema__ = 'https://127.0.0.1:5000/schemas/series/series-v1.0.0.json' + __schema__ = "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json" __model_ignore_keys__ = set() __ignore_keys__ = COMMON_IGNORE_FIELDS | __model_ignore_keys__ - __json_init_dict__ = {'_migration': {**get_migration_dict()}} + __json_init_dict__ = {"_migration": {**get_migration_dict()}} def do(self, blob, ignore_missing=True, exception_handlers=None): """Set schema after translation depending on the model.""" @@ -47,16 +50,21 @@ def do(self, blob, ignore_missing=True, exception_handlers=None): blob=blob, ignore_missing=ignore_missing, exception_handlers=exception_handlers, - init_fields=deepcopy(self.__json_init_dict__)) - json['$schema'] = self.__class__.__schema__ + init_fields=deepcopy(self.__json_init_dict__), + ) + json["$schema"] = self.__class__.__schema__ - json['_migration'].update({'record_type': 'multipart'}) + json["_migration"].update({"record_type": "multipart"}) - json['mode_of_issuance'] = 'MULTIPART_MONOGRAPH' + json["mode_of_issuance"] = "MULTIPART_MONOGRAPH" return json model = CDSMultipart( - bases=(books_base, cds_base, ), - entry_point_group='cds_dojson.marc21.series') + bases=( + books_base, + cds_base, + ), + entry_point_group="cds_dojson.marc21.series", +) diff --git a/cds_dojson/marc21/models/books/serial.py b/cds_dojson/marc21/models/books/serial.py index a98dd338..1669a66a 100644 --- a/cds_dojson/marc21/models/books/serial.py +++ b/cds_dojson/marc21/models/books/serial.py @@ -17,6 +17,7 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Book model.""" + from __future__ import unicode_literals from ..base import model as cds_base @@ -27,179 +28,181 @@ class CDSSerial(CDSOverdoBookBase): """Translation Index for CDS Books.""" - __query__ = '(690C_:BOOK OR 690C_:"YELLOW REPORT" OR ' \ - '980__:PROCEEDINGS OR ' \ - '697C_:LEGSERLIB -980__:DELETED -980__:MIGRATED)' \ - ' AND 490__:/[a-zA-Z0-9]+/' + __query__ = ( + '(690C_:BOOK OR 690C_:"YELLOW REPORT" OR ' + "980__:PROCEEDINGS OR " + "697C_:LEGSERLIB -980__:DELETED -980__:MIGRATED)" + " AND 490__:/[a-zA-Z0-9]+/" + ) - __schema__ = 'https://127.0.0.1:5000/schemas/series/series-v1.0.0.json' + __schema__ = "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json" __model_ignore_keys__ = { - '020__a', - '020__b', - '020__c', - '020__u', - '022__a', - '022__b', - '0247_2', - '0247_9', - '0247_a', - '0247_q', - '0247_y', - '0248_q', - '035__9', - '035__a', - '035__d', - '035__h', - '035__m', - '035__t', - '035__u', - '035_a9', - '035_aa', - '037__9', - '037__a', - '037__c', - '041__a', - '044__a', - '050_4a', - '050__a', - '080__a', - '082042', - '08204a', - '082__a', - '084__2', - '084__a', - '088__9', - '088__a', - '100__a', - '100__e', - '100__u', - '100__9', - '110__a', - '111__9', - '111__a', - '111__c', - '111__g', - '111__n', - '111__w', - '111__y', - '111__z', - '210__a', - '222__a', - '242__a', - '245__9', - '245__a', - '245__b', - '245__c', - '246__a', - '246__i', - '246__n', - '246__p', - '250__a', - '260__a', - '260__b', - '260__c', - '270__a', - '270__d', - '270__k', - '270__l', - '270__m', - '300__a', - '300__b', - '310__a', - '4901_a', - '490__v', - '500__9', - '500__a', - '5050_a', - '505__a', - '505__t', - '520__9', - '520__a', - '536__a', - '536__c', - '536__f', - '536__r', - '540__3', - '540__a', - '540__u', - '541__9', - '542__3', - '542__d', - '542__g', - '583__c', - '595__9', - '595__a', - '595__i', - '595__z', - '65027b', - '6531_9', - '6531_a', - '690C_a', - '690c_a', - '693__a', - '693__e', - '695__9', - '695__a', - '697C_a', - '700__0', - '700__9', - '700__a', - '700__e', - '700__i', - '700__u', - '710__5', - '710__a', - '710__e', - '710__g', - '711__9', - '711__a', - '711__c', - '711__d', - '711__f', - '711__g', - '711__n', - '711__w', - '711__z', - '773__c', - '773__n', - '773__p', - '773__v', - '773__w', - '773__y', - '773__y', - '775__a', - '775__b', - '775__c', - '775__w', - '852__p', - '8564_8', - '8564_s', - '8564_t', - '8564_u', - '8564_w', - '8564_x', - '8564_y', - '859__f', - '901__u', - '912__f', - '912__r', - '916__a', - '916__s', - '916__w', - '933__a', - '933__b', - '938__a', - '938__p', - '960__a', - '962__k', - '970__a', - '970__d', - '980__a', - '980__b', - '980__c', - '993__r', - '999C6a', + "020__a", + "020__b", + "020__c", + "020__u", + "022__a", + "022__b", + "0247_2", + "0247_9", + "0247_a", + "0247_q", + "0247_y", + "0248_q", + "035__9", + "035__a", + "035__d", + "035__h", + "035__m", + "035__t", + "035__u", + "035_a9", + "035_aa", + "037__9", + "037__a", + "037__c", + "041__a", + "044__a", + "050_4a", + "050__a", + "080__a", + "082042", + "08204a", + "082__a", + "084__2", + "084__a", + "088__9", + "088__a", + "100__a", + "100__e", + "100__u", + "100__9", + "110__a", + "111__9", + "111__a", + "111__c", + "111__g", + "111__n", + "111__w", + "111__y", + "111__z", + "210__a", + "222__a", + "242__a", + "245__9", + "245__a", + "245__b", + "245__c", + "246__a", + "246__i", + "246__n", + "246__p", + "250__a", + "260__a", + "260__b", + "260__c", + "270__a", + "270__d", + "270__k", + "270__l", + "270__m", + "300__a", + "300__b", + "310__a", + "4901_a", + "490__v", + "500__9", + "500__a", + "5050_a", + "505__a", + "505__t", + "520__9", + "520__a", + "536__a", + "536__c", + "536__f", + "536__r", + "540__3", + "540__a", + "540__u", + "541__9", + "542__3", + "542__d", + "542__g", + "583__c", + "595__9", + "595__a", + "595__i", + "595__z", + "65027b", + "6531_9", + "6531_a", + "690C_a", + "690c_a", + "693__a", + "693__e", + "695__9", + "695__a", + "697C_a", + "700__0", + "700__9", + "700__a", + "700__e", + "700__i", + "700__u", + "710__5", + "710__a", + "710__e", + "710__g", + "711__9", + "711__a", + "711__c", + "711__d", + "711__f", + "711__g", + "711__n", + "711__w", + "711__z", + "773__c", + "773__n", + "773__p", + "773__v", + "773__w", + "773__y", + "773__y", + "775__a", + "775__b", + "775__c", + "775__w", + "852__p", + "8564_8", + "8564_s", + "8564_t", + "8564_u", + "8564_w", + "8564_x", + "8564_y", + "859__f", + "901__u", + "912__f", + "912__r", + "916__a", + "916__s", + "916__w", + "933__a", + "933__b", + "938__a", + "938__p", + "960__a", + "962__k", + "970__a", + "970__d", + "980__a", + "980__b", + "980__c", + "993__r", + "999C6a", } __ignore_keys__ = COMMON_IGNORE_FIELDS | __model_ignore_keys__ @@ -209,14 +212,11 @@ def do(self, blob, ignore_missing=True, exception_handlers=None): json = super(CDSSerial, self).do( blob=blob, ignore_missing=ignore_missing, - exception_handlers=exception_handlers) - json['$schema'] = self.__class__.__schema__ - json['_migration'] = {'record_type': 'serial', - 'children': [] - } + exception_handlers=exception_handlers, + ) + json["$schema"] = self.__class__.__schema__ + json["_migration"] = {"record_type": "serial", "children": []} return json -model = CDSSerial( - bases=(cds_base,), - entry_point_group='cds_dojson.marc21.series') +model = CDSSerial(bases=(cds_base,), entry_point_group="cds_dojson.marc21.series") diff --git a/cds_dojson/marc21/models/books/standard.py b/cds_dojson/marc21/models/books/standard.py index f508925d..e0e34584 100644 --- a/cds_dojson/marc21/models/books/standard.py +++ b/cds_dojson/marc21/models/books/standard.py @@ -30,13 +30,13 @@ class CDSStandard(CDSOverdoBookBase): """Translation Index for CDS Books.""" - __query__ = '690C_:STANDARD OR 980__:STANDARD -980__:DELETED -980__:MIGRATED' + __query__ = "690C_:STANDARD OR 980__:STANDARD -980__:DELETED -980__:MIGRATED" - __schema__ = 'https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json' + __schema__ = "https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json" __ignore_keys__ = COMMON_IGNORE_FIELDS - __json_init_dict__ = {'_migration': {**get_migration_dict()}} + __json_init_dict__ = {"_migration": {**get_migration_dict()}} def do(self, blob, ignore_missing=True, exception_handlers=None): """Set schema after translation depending on the model.""" @@ -47,10 +47,14 @@ def do(self, blob, ignore_missing=True, exception_handlers=None): init_fields=deepcopy(self.__json_init_dict__), ) - json['$schema'] = self.__class__.__schema__ + json["$schema"] = self.__class__.__schema__ return json model = CDSStandard( - bases=(books_base, cds_base,), - entry_point_group='cds_dojson.marc21.book') + bases=( + books_base, + cds_base, + ), + entry_point_group="cds_dojson.marc21.book", +) diff --git a/cds_dojson/marc21/models/videos/base.py b/cds_dojson/marc21/models/videos/base.py index 21342c6b..aad3e108 100644 --- a/cds_dojson/marc21/models/videos/base.py +++ b/cds_dojson/marc21/models/videos/base.py @@ -26,5 +26,4 @@ class VideosBase(OverdoJSONSchema): """Base model conversion MARC21 to JSON.""" -model = VideosBase(bases=(cds_base, ), - entry_point_group='cds_dojson.marc21.videos') +model = VideosBase(bases=(cds_base,), entry_point_group="cds_dojson.marc21.videos") diff --git a/cds_dojson/marc21/models/videos/project.py b/cds_dojson/marc21/models/videos/project.py index 53a2ed45..7d1bddee 100644 --- a/cds_dojson/marc21/models/videos/project.py +++ b/cds_dojson/marc21/models/videos/project.py @@ -26,31 +26,35 @@ class CDSVideoProject(OverdoJSONSchema): """Translation Index for CDS Video Projects.""" - __query__ = '970__:project -980__:DELETED' + __query__ = "970__:project -980__:DELETED" - __schema__ = 'records/videos/project/project-v1.0.0.json' + __schema__ = "records/videos/project/project-v1.0.0.json" __ignore_keys__ = { - '005', - '260__a', - '260__c', - '269__c', - '5061_a', - '690C_a', - '774__o', - '774__r', - '937__c', - '960__a', - '961__c', - '961__h', - '961__l', - '961__x', - '980__a', - '980__b', - '981__a', + "005", + "260__a", + "260__c", + "269__c", + "5061_a", + "690C_a", + "774__o", + "774__r", + "937__c", + "960__a", + "961__c", + "961__h", + "961__l", + "961__x", + "980__a", + "980__b", + "981__a", } model = CDSVideoProject( - bases=(videos_base, cds_base, ), - entry_point_group='cds_dojson.marc21.video') + bases=( + videos_base, + cds_base, + ), + entry_point_group="cds_dojson.marc21.video", +) diff --git a/cds_dojson/marc21/models/videos/video.py b/cds_dojson/marc21/models/videos/video.py index a18ab4a9..029bd1ea 100644 --- a/cds_dojson/marc21/models/videos/video.py +++ b/cds_dojson/marc21/models/videos/video.py @@ -26,70 +26,74 @@ class CDSVideo(OverdoJSONSchema): """Translation Index for CDS Videos.""" - __query__ = '980__:VIDEO -980__:VIDEOARC -980__:VIDEOSHOOT \ - -970__:project -980__:DELETED' + __query__ = "980__:VIDEO -980__:VIDEOARC -980__:VIDEOSHOOT \ + -970__:project -980__:DELETED" - __schema__ = 'records/videos/video/video-v1.0.0.json' + __schema__ = "records/videos/video/video-v1.0.0.json" __ignore_keys__ = { - '005', - '0248_a', - '0248_p', - '0248_q', - '035__9', - '035__a', - '100__9', - '260__c', - '269__b', - '300__b', - '300__c', - '300__d', - '300__e', - '337__a', - '5061_2', - '5061_5', - '5061_a', - '5061_f', - '5061_z', - '542__e', - '690C_a', - '700__0', - '700__9', - '773__o', - '773__r', - '787__i', - '787__w', - '852__j', - '852__x', + "005", + "0248_a", + "0248_p", + "0248_q", + "035__9", + "035__a", + "100__9", + "260__c", + "269__b", + "300__b", + "300__c", + "300__d", + "300__e", + "337__a", + "5061_2", + "5061_5", + "5061_a", + "5061_f", + "5061_z", + "542__e", + "690C_a", + "700__0", + "700__9", + "773__o", + "773__r", + "787__i", + "787__w", + "852__j", + "852__x", # FIXME need to double check (see #85) - '8564_8', - '8564_d', - '8564_q', - '8564_s', - '8564_u', - '8564_x', - '8564_y', - '8564_z', - '8567_2', - '916__s', - '916__w', - '937__c', - '960__a', - '961__c', - '961__h', - '961__l', - '961__x', - '962__b', - '962__l', - '962__n', - '962__t', - '963__a', - '980__a', - '980__b', - '981__a', + "8564_8", + "8564_d", + "8564_q", + "8564_s", + "8564_u", + "8564_x", + "8564_y", + "8564_z", + "8567_2", + "916__s", + "916__w", + "937__c", + "960__a", + "961__c", + "961__h", + "961__l", + "961__x", + "962__b", + "962__l", + "962__n", + "962__t", + "963__a", + "980__a", + "980__b", + "981__a", } model = CDSVideo( - bases=(videos_base, cds_base, ), - entry_point_group='cds_dojson.marc21.video') + bases=( + videos_base, + cds_base, + ), + entry_point_group="cds_dojson.marc21.video", +) diff --git a/cds_dojson/marc21/utils.py b/cds_dojson/marc21/utils.py index 9810ddb3..c74c7e0d 100644 --- a/cds_dojson/marc21/utils.py +++ b/cds_dojson/marc21/utils.py @@ -32,56 +32,57 @@ def create_record(marcxml, correct=False, keep_singletons=True): If correct == 0, then do not perform DTD validation """ if isinstance(marcxml, binary_type): - marcxml = marcxml.decode('utf-8') + marcxml = marcxml.decode("utf-8") if isinstance(marcxml, text_type): parser = etree.XMLParser(dtd_validation=correct, recover=True) if correct: - marcxml = (u'\n' - u'\n' - u'\n{1}\n'.format( - MARC21_DTD, marcxml)) + marcxml = ( + '\n' + '\n' + "\n{1}\n".format(MARC21_DTD, marcxml) + ) tree = etree.parse(StringIO(marcxml), parser) else: tree = marcxml record = [] - leader_iterator = tree.iter(tag='{*}leader') + leader_iterator = tree.iter(tag="{*}leader") for leader in leader_iterator: - text = leader.text or '' - record.append(('leader', text)) + text = leader.text or "" + record.append(("leader", text)) - controlfield_iterator = tree.iter(tag='{*}controlfield') + controlfield_iterator = tree.iter(tag="{*}controlfield") for controlfield in controlfield_iterator: - tag = controlfield.attrib.get('tag', '!') - text = controlfield.text or '' + tag = controlfield.attrib.get("tag", "!") + text = controlfield.text or "" if text or keep_singletons: record.append((tag, text)) - datafield_iterator = tree.iter(tag='{*}datafield') + datafield_iterator = tree.iter(tag="{*}datafield") for datafield in datafield_iterator: - tag = datafield.attrib.get('tag', '!') - ind1 = datafield.attrib.get('ind1', '!') - ind2 = datafield.attrib.get('ind2', '!') - if ind1 in ('', '#'): - ind1 = '_' - if ind2 in ('', '#'): - ind2 = '_' - ind1 = ind1.replace(' ', '_') - ind2 = ind2.replace(' ', '_') + tag = datafield.attrib.get("tag", "!") + ind1 = datafield.attrib.get("ind1", "!") + ind2 = datafield.attrib.get("ind2", "!") + if ind1 in ("", "#"): + ind1 = "_" + if ind2 in ("", "#"): + ind2 = "_" + ind1 = ind1.replace(" ", "_") + ind2 = ind2.replace(" ", "_") fields = [] - subfield_iterator = datafield.iter(tag='{*}subfield') + subfield_iterator = datafield.iter(tag="{*}subfield") for subfield in subfield_iterator: - code = subfield.attrib.get('code', '!') # .encode("UTF-8") - text = subfield.text or '' + code = subfield.attrib.get("code", "!") # .encode("UTF-8") + text = subfield.text or "" if text or keep_singletons: fields.append((code, text)) if fields or keep_singletons: - key = '{0}{1}{2}'.format(tag, ind1, ind2) + key = "{0}{1}{2}".format(tag, ind1, ind2) record.append((key, MementoDict(fields))) return MementoDict(record) diff --git a/cds_dojson/matcher.py b/cds_dojson/matcher.py index a4e8a42f..9640e9e4 100644 --- a/cds_dojson/matcher.py +++ b/cds_dojson/matcher.py @@ -20,19 +20,23 @@ """Query parser.""" import logging +import sys -import importlib_metadata import pypeg2 -from dojson.contrib.marc21 import model as default from invenio_query_parser.parser import Main as parser from invenio_query_parser.walkers.match_unit import MatchUnit from invenio_query_parser.walkers.pypeg_to_ast import PypegConverter from cds_dojson.exceptions import ModelMissingException, MultipleModelsException +if sys.version_info >= (3, 10): + from importlib.metadata import entry_points +else: # pragma: no cover + from importlib_metadata import entry_points + # Cache of (name, model, parsed_query_ast) tuples keyed by entry_point_group. # Entry points and their query ASTs are static for the lifetime of a process, -# so scanning importlib_metadata and re-parsing pypeg2 queries on every call +# so scanning importlib.metadata and re-parsing pypeg2 queries on every call # was the dominant cost (~84s for 1320 calls). _models_cache = {} @@ -61,7 +65,7 @@ def match(self, record, user_info=None): def _load_models(entry_point_group): """Load and cache entry point models for a given group.""" if entry_point_group not in _models_cache: - entrypoints = set(importlib_metadata.entry_points(group=entry_point_group)) + entrypoints = set(entry_points(group=entry_point_group)) models = [] for ep in entrypoints: model = ep.load() @@ -79,32 +83,25 @@ def matcher(record, entry_point_group): :param record: Something that looks like a python dictionary :returns: a model instance + :raises ModelMissingException: if no model matches the record + :raises MultipleModelsException: if more than one model matches the record """ logger = logging.getLogger(__name__ + ".dojson_matcher") _matches = [] for name, model, query in _load_models(entry_point_group): if query.match(record): - logger.info("Model `{0}` found matching the query {1}.".format( - name, model - )) + logger.info("Model `{0}` found matching the query {1}.".format(name, model)) _matches.append([name, model]) if len(_matches) == 1: return _matches[0][1] elif len(_matches) > 1: logger.error( - ("Found more than one matches `{0}`, we'll use {1}" - " for record {2}.").format( - _matches, default, record - ) + "Found more than one match {0} for record {1}.".format(_matches, record) ) raise MultipleModelsException( - "Found more than one models {0} for record {1}".format( - _matches, record)) - elif len(_matches) == 0: - logger.warning( - "Model *not* found, fallback to default {0} for record {1}".format( - default, record - ) + "Found more than one models {0} for record {1}".format(_matches, record) ) + else: + logger.warning("Model *not* found for record {0}".format(record)) raise ModelMissingException("Model *not* found") diff --git a/cds_dojson/overdo.py b/cds_dojson/overdo.py index 404dab07..1624f913 100644 --- a/cds_dojson/overdo.py +++ b/cds_dojson/overdo.py @@ -25,7 +25,7 @@ from .utils import not_accessed_keys try: - importlib_metadata.distribution('flask') + importlib_metadata.distribution("flask") from flask import current_app except (importlib_metadata.PackageNotFoundError, RuntimeError) as e: HAS_FLASK = False @@ -36,10 +36,7 @@ class OverdoBase(DoJSONOverdo): """Base entry class.""" - def __init__(self, - bases=None, - entry_point_group=None, - entry_point_models=None): + def __init__(self, bases=None, entry_point_group=None, entry_point_models=None): """Init.""" super(OverdoBase, self).__init__(bases, entry_point_group) self.entry_point_models = entry_point_models @@ -60,7 +57,7 @@ def missing(self, blob, **kwargs): class Overdo(DoJSONOverdo): """Translation index base.""" - __query__ = '' + __query__ = "" """To be used by the matcher to find the proper model.""" __ignore_keys__ = set() @@ -74,6 +71,7 @@ def over(self, name, *source_tags, **kwargs): regular expression in `source_tags` are equal to the current ones. """ + def override(rule): if name == rule[1][0]: return True @@ -82,7 +80,7 @@ def override(rule): return True return False - if kwargs.get('override', False): + if kwargs.get("override", False): self.rules[:] = [rule for rule in self.rules if not override(rule)] return super(Overdo, self).over(name, *source_tags) @@ -95,7 +93,7 @@ def missing(self, blob, **kwargs): class OverdoJSONSchema(Overdo): """Translation index which adds $schema key.""" - __schema__ = '' + __schema__ = "" """Name of the schema to be added to the final JSON.""" def do(self, blob, ignore_missing=True, exception_handlers=None): @@ -103,13 +101,14 @@ def do(self, blob, ignore_missing=True, exception_handlers=None): json = super(OverdoJSONSchema, self).do( blob=blob, ignore_missing=ignore_missing, - exception_handlers=exception_handlers) + exception_handlers=exception_handlers, + ) if HAS_FLASK: - json_schema = current_app.extensions['invenio-jsonschemas'] - json['$schema'] = { - '$ref': json_schema.path_to_url(self.__class__.__schema__) + json_schema = current_app.extensions["invenio-jsonschemas"] + json["$schema"] = { + "$ref": json_schema.path_to_url(self.__class__.__schema__) } else: - json['$schema'] = {'$ref': self.__class__.__schema__} + json["$schema"] = {"$ref": self.__class__.__schema__} return json diff --git a/cds_dojson/schemas/transform.py b/cds_dojson/schemas/transform.py index 97dd0ab0..58149b03 100644 --- a/cds_dojson/schemas/transform.py +++ b/cds_dojson/schemas/transform.py @@ -29,7 +29,7 @@ from jsonref import JsonRef -def compile_schema(schema, base_uri='', in_place=False): +def compile_schema(schema, base_uri="", in_place=False): """Resolve references in schema. It is assumed that schemas have already been validated for backward @@ -50,8 +50,8 @@ def compile_schema(schema, base_uri='', in_place=False): def _compile_all_of(schema): """Go through the schema an compile the ``allOf`` rules.""" if isinstance(schema, dict): - if 'allOf' in schema: - all_of = schema.pop('allOf') + if "allOf" in schema: + all_of = schema.pop("allOf") schema = merge_dicts(schema, *all_of) for key, value in six.iteritems(schema): schema[key] = _compile_all_of(value) diff --git a/cds_dojson/utils.py b/cds_dojson/utils.py index 7d93307d..b7288b2f 100644 --- a/cds_dojson/utils.py +++ b/cds_dojson/utils.py @@ -21,6 +21,7 @@ import functools import json import os + try: from collections import defaultdict except ImportError: @@ -39,11 +40,13 @@ class MementoDict(GroupableOrderedDict): def __new__(cls, *args): """Add the memory to the default instance.""" cls.accessed_keys = property( - lambda self: set([k for k in self.__memory if k != '__order__'])) + lambda self: set([k for k in self.__memory if k != "__order__"]) + ) cls.not_accessed_keys = property( - lambda self: set( - [k for k in self.keys() if k != '__order__'] - ).difference(self.__memory)) + lambda self: set([k for k in self.keys() if k != "__order__"]).difference( + self.__memory + ) + ) new = GroupableOrderedDict.__new__(cls, *args) new.__memory = set() new.__skip_memento = False @@ -61,10 +64,12 @@ def iteritems(self, skip_memento=False, **kwargs): def __repr__(self): """Output the representation of the GroupableOrderedDict.""" - out = ("({!r}, {!r})".format(k, v) - for k, v in self.iteritems(skip_memento=True, repeated=True) - if k != '__order__') - return 'GroupableOrderedDict(({out}))'.format(out=', '.join(out)) + out = ( + "({!r}, {!r})".format(k, v) + for k, v in self.iteritems(skip_memento=True, repeated=True) + if k != "__order__" + ) + return "GroupableOrderedDict(({out}))".format(out=", ".join(out)) def _add_to_memory(self, key): """Add key to the memory is it is not locked.""" @@ -89,6 +94,7 @@ def for_each_squash(f): [{'a': 'foo'}, {'b': 'bar'}] -> {'a': 'foo', 'b': 'barc'} [{'a': 'foo'}, {'a': 'bar'}] -> {'a': ['foo', 'barc']} """ + @functools.wraps(f) def wrapper(self, key, values, **kwargs): if not isinstance(values, list): @@ -121,8 +127,7 @@ def not_accessed_keys(blob): for key, value in blob.iteritems(skip_memento=True): partial_missing = not_accessed_keys(value) if partial_missing: - missing.update( - ['{0}{1}'.format(key, f) for f in partial_missing]) + missing.update(["{0}{1}".format(key, f) for f in partial_missing]) if key in missing: missing.remove(key) elif isinstance(blob, (tuple, list)): @@ -132,7 +137,7 @@ def not_accessed_keys(blob): return missing -def convert_date_to_iso_8601(date, format_='YYYY-MM-DD', **kwargs): +def convert_date_to_iso_8601(date, format_="YYYY-MM-DD", **kwargs): """Convert a date string its ISO 8601 representation. YYYY-MM-DDThh:mm:ss.sTZD (eg 1997-07-16T19:20:30.45+01:00) @@ -148,26 +153,27 @@ def convert_date_to_iso_8601(date, format_='YYYY-MM-DD', **kwargs): """ # The order is important as arrow tries to apply them top to bottom _FORMATS = [ - 'YYYY-MM-DD', - 'YYYY/MM/DD', - 'DD/MM/YYYY', - 'DD/MM/YY', - 'YYYY.MM.DD', - 'DD MMMM YYYY', - 'DD MMM YYYY', - 'DD MMM YY', - 'YYYY-MM', - 'YYYY/MM', - 'YYYY.MM', - 'MMM YYYY', - 'YYYY', - 'YY', + "YYYY-MM-DD", + "YYYY/MM/DD", + "DD/MM/YYYY", + "DD/MM/YY", + "YYYY.MM.DD", + "DD MMMM YYYY", + "DD MMM YYYY", + "DD MMM YY", + "YYYY-MM", + "YYYY/MM", + "YYYY.MM", + "MMM YYYY", + "YYYY", + "YY", ] return arrow.get(date, _FORMATS).format(format_) if date else date def yaml2json(source, destination): """Converts recursively yaml files to json.""" + def _get_current_directory_name(root, source): """Reurns the current directory name. @@ -176,31 +182,30 @@ def _get_current_directory_name(root, source): source: './ymls' return: 'elements' """ - return root.replace(os.path.commonprefix([root, source]), '') + return root.replace(os.path.commonprefix([root, source]), "") for root, directories, filenames in os.walk(source): for directory in directories: destination_directory = os.path.join( - destination, _get_current_directory_name(root, source), - directory) + destination, _get_current_directory_name(root, source), directory + ) if not os.path.exists(destination_directory): os.makedirs(destination_directory) for filename in filenames: filename_ext = os.path.splitext(filename)[1] - if filename_ext == '.yml' or filename_ext == '.yaml': - new_filename = os.path.splitext(filename)[0] + '.json' + if filename_ext == ".yml" or filename_ext == ".yaml": + new_filename = os.path.splitext(filename)[0] + ".json" directory = _get_current_directory_name(root, source) read_file_path = os.path.join(root, filename) - write_file_path = os.path.join(destination, directory, - new_filename) + write_file_path = os.path.join(destination, directory, new_filename) with open(read_file_path) as stream: # read the yml file yaml_data = yaml.safe_load(stream) - with open(write_file_path, 'w') as f: + with open(write_file_path, "w") as f: # write the json file json.dump(yaml_data, f, indent=2) f.write("\n") diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 2836bbed..00000000 --- a/pytest.ini +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of DoJSON -# Copyright (C) 2014-2021 CERN. -# -# DoJSON is free software; you can redistribute it and/or modify -# it under the terms of the Revised BSD License; see LICENSE file for -# more details. - -[pytest] -addopts = --isort --doctest-glob="*.rst" --doctest-modules --cov=cds_dojson --cov-report=term-missing -testpaths = tests cds_dojson diff --git a/run-tests.sh b/run-tests.sh index e4b2c23f..b2be02cc 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -24,6 +24,5 @@ set -o errexit # Quit on unbound symbols set -o nounset -pydocstyle cds_dojson tests && \ python -m check_manifest --ignore ".*-requirements.txt" && \ python -m pytest tests diff --git a/setup.cfg b/setup.cfg index 873d2359..97828233 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,8 @@ zip_safe = False install_requires = arrow>=0.7.0 dojson>=1.3.2 - invenio-query-parser>=0.5.0 + importlib_metadata>=4.6; python_version < "3.10" + invenio-query-parser>=0.7.0 requests>=2.17.3 pycountry>=17.5.14 PyYAML>=3.13 @@ -24,26 +25,16 @@ install_requires = [options.extras_require] tests = - check-manifest>=0.25 - coverage>=5.3,<6 - invenio-jsonschemas>=1.0.0a5,<1.1.2 + pytest-invenio>=4.0.0,<5.0.0 + pytest-black-ng>=0.4.0 + sphinx>=5 + invenio-jsonschemas>=1.1.3 jsonpatch>=1.11 jsonref>=0.1 jsonresolver>=0.1.0 jsonschema>=2.5.1 - mock>=1.3.0 - pytest-runner>=2.6.2 - pydocstyle>=1.0.0 - pycodestyle>=1.0.0 - pytest-cache>=1.0 - pytest-cov>=2.10.1 - pytest-isort>=1.2.0 - pytest-pep8>=1.0.6 - pytest>=4.0.0 - pycountry>=17.5.14 - Sphinx>=1.3 docs = - Sphinx>=1.3 + Sphinx>=5 [options.entry_points] cds_dojson.marc21.parent_models = @@ -87,9 +78,6 @@ source-dir = docs/ build-dir = docs/_build all_files = 1 -[bdist_wheel] -universal = 1 - [compile_catalog] directory = cds_dojson/translations/ @@ -112,16 +100,17 @@ output-dir = cds_dojson/translations/ add_ignore = D401,D403 [isort] -profile=black +profile = black + +[pycodestyle] +max-line-length = 88 +ignore = E203,W503 +exclude = docs/conf.py,tests/* [check-manifest] ignore = *-requirements.txt [tool:pytest] -addopts = --black --isort --pydocstyle --doctest-glob="*.rst" --doctest-modules --cov=cds_dojson --cov-report=term-missing +addopts = --black --isort --pycodestyle --pydocstyle --doctest-glob="*.rst" --doctest-modules --cov=cds_dojson --cov-report=term-missing testpaths = tests cds_dojson -live_server_scope = module - -[pycodestyle] -exclude = docs/conf.py,tests/* diff --git a/tests/conftest.py b/tests/conftest.py index 15d33d11..abdcde10 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,7 @@ def app(): app_ = Flask(__name__) app_.config.update( TESTING=True, - JSONSCHEMAS_HOST='cds.cern.ch', + JSONSCHEMAS_HOST="cds.cern.ch", ) app_.config.update(TESTING=True) diff --git a/tests/demo/json_resolver.py b/tests/demo/json_resolver.py index 1d7fc29f..6c834917 100644 --- a/tests/demo/json_resolver.py +++ b/tests/demo/json_resolver.py @@ -24,18 +24,14 @@ """Test JSON resolver.""" -from __future__ import absolute_import, print_function - import json +from importlib.resources import files import jsonresolver -import pkg_resources -@jsonresolver.route('/schemas/', - host='cds.cern.ch') +@jsonresolver.route("/schemas/", host="cds.cern.ch") def resolve_definitions(path): """Resolve the JSON definition schema.""" - with open(pkg_resources.resource_filename( - 'cds_dojson.schemas', path), 'r') as f: + with (files("cds_dojson.schemas") / path).open("r") as f: return json.load(f) diff --git a/tests/helpers.py b/tests/helpers.py index 33b2e2eb..23e940f5 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -18,28 +18,28 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Test utilities.""" -import os from itertools import chain +from pathlib import Path -import pkg_resources from jsonresolver import JSONResolver from jsonresolver.contrib.jsonschema import ref_resolver_factory from jsonschema import validate as _validate -from six import iteritems from cds_dojson.utils import MementoDict def mock_path_to_url(self, path): - """Mock the ``paht_to_utl method from InvenioJSONSchemasState.""" + """Mock the ``path_to_url`` method from InvenioJSONSchemasState.""" return path def json_resolver(schema): """Test ``json_resolver``.""" - json_resolver = JSONResolver(plugins=[ - 'demo.json_resolver', - ]) + json_resolver = JSONResolver( + plugins=[ + "demo.json_resolver", + ] + ) resolver_cls = ref_resolver_factory(json_resolver) return resolver_cls.from_schema(schema) @@ -48,43 +48,40 @@ def validate(json): """Wrap jsonchema.validate for convenience.""" return _validate( json, - json['$schema'], - resolver=json_resolver(json['$schema']), + json["$schema"], + resolver=json_resolver(json["$schema"]), # types={'array': (list, tuple)} ) def load_fixture_file(file_name): """Read the content of a file and return it.""" - return pkg_resources.resource_string(__name__, - os.path.join('fixtures', - file_name)) + return (Path(__file__).parent / "fixtures" / file_name).read_bytes() def mock_contributor_fetch(info): """Mock contributor fetch.""" - name = info.get('a') + name = info.get("a") author_info = dict() - if name == 'Test User': + if name == "Test User": author_info = dict( - name='User, Test', - lastname='User', - recid='123456789', - email='tuser@cern.ch', - firstname='Test', - cernccid='987654', + name="User, Test", + lastname="User", + recid="123456789", + email="tuser@cern.ch", + firstname="Test", + cernccid="987654", ) - elif name == 'Test User 2': + elif name == "Test User 2": author_info = dict( - name='User, Test 2', - lastname='User 2', - recid='2123456789', - email='tuser2@cern.ch', - firstname='Test 2', - cernccid='9876542', + name="User, Test 2", + lastname="User 2", + recid="2123456789", + email="tuser2@cern.ch", + firstname="Test 2", + cernccid="9876542", ) else: return info - return MementoDict([ - (k, v) for k, v in chain(info.items(), iteritems(author_info))]) + return MementoDict(list(chain(info.items(), author_info.items()))) diff --git a/tests/test_base.py b/tests/test_base.py index 0d33dee0..21357956 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -18,10 +18,8 @@ # 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA. """Base model tests.""" -import os - -import mock -import pkg_resources +from pathlib import Path +from unittest import mock from cds_dojson.marc21.models.base import model from cds_dojson.marc21.utils import create_record @@ -30,27 +28,27 @@ def test_base_model(app): """Test base model.""" - marcxml = pkg_resources.resource_string(__name__, - os.path.join( - 'fixtures', 'base.xml')) + marcxml = (Path(__file__).parent / "fixtures" / "base.xml").read_bytes() with app.app_context(): blob = create_record(marcxml) - assert model.missing(blob) == {'001', '003', '005'} + assert model.missing(blob) == {"001", "003", "005"} record = model.do(blob) - assert record['recid'] == 1495143 - assert record['agency_code'] == 'SzGeCERN' + assert record["recid"] == 1495143 + assert record["agency_code"] == "SzGeCERN" assert not model.missing(blob) def test_overdo_missing(): """Test missing check from Overdo.""" + class MyOverdo(Overdo): - __ignore_keys__ = set(['c']) + __ignore_keys__ = {"c"} - with mock.patch('cds_dojson.overdo.not_accessed_keys', - return_value=set(['a', 'b', 'c'])): + with mock.patch( + "cds_dojson.overdo.not_accessed_keys", return_value={"a", "b", "c"} + ): overdo = MyOverdo() - overdo.__ignore_keys__ = set(['c']) - assert overdo.missing('blob') == set(['a', 'b']) + overdo.__ignore_keys__ = {"c"} + assert overdo.missing("blob") == {"a", "b"} diff --git a/tests/test_books.py b/tests/test_books.py index d2b8a8c3..922940a6 100644 --- a/tests/test_books.py +++ b/tests/test_books.py @@ -35,21 +35,23 @@ from cds_dojson.marc21.models.books.book import model from cds_dojson.marc21.utils import create_record -marcxml = ("""""" - """{0}""") +marcxml = ( + """""" + """{0}""" +) def check_transformation(marcxml_body, json_body): """Check transformation.""" blob = create_record(marcxml.format(marcxml_body)) - record = {'_migration': {**get_migration_dict()}} + record = {"_migration": {**get_migration_dict()}} record.update(**model.do(blob, ignore_missing=False)) expected = { - '$schema': 'https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json', - '_migration': {**get_migration_dict()} + "$schema": "https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json", + "_migration": {**get_migration_dict()}, } expected.update(**json_body) @@ -59,7 +61,7 @@ def check_transformation(marcxml_body, json_body): def test_mapping(): """Test mapping.""" with pytest.raises(UnexpectedValue): - assert mapping(MATERIALS, 'softwa', raise_exception=True) == 'software' + assert mapping(MATERIALS, "softwa", raise_exception=True) == "software" def test_subject_classification(app): @@ -71,36 +73,27 @@ def test_subject_classification(app): 515.353 23 - """, { - 'subjects': [ - {'value': '515.353', - 'scheme': 'Dewey'} - ] - }) + """, + {"subjects": [{"value": "515.353", "scheme": "Dewey"}]}, + ) check_transformation( """ QA76.642 XXXX - """, { - 'subjects': [ - {'value': 'QA76.642', - 'scheme': 'LoC'} - ] - }) + """, + {"subjects": [{"value": "QA76.642", "scheme": "LoC"}]}, + ) check_transformation( """ QA76.642 XXXX - """, { - 'subjects': [ - {'value': 'QA76.642', - 'scheme': 'LoC'} - ] - }) + """, + {"subjects": [{"value": "QA76.642", "scheme": "LoC"}]}, + ) check_transformation( """ @@ -109,37 +102,30 @@ def test_subject_classification(app): 005.275 - """, { - 'subjects': [ - {'value': 'QA76.642', - 'scheme': 'LoC'}, - {'value': '005.275', - 'scheme': 'Dewey'}, + """, + { + "subjects": [ + {"value": "QA76.642", "scheme": "LoC"}, + {"value": "005.275", "scheme": "Dewey"}, ] - }) + }, + ) check_transformation( """ 528 - """, { - 'subjects': [ - {'value': '528', - 'scheme': 'UDC'} - ] - }) + """, + {"subjects": [{"value": "528", "scheme": "UDC"}]}, + ) check_transformation( """ 25040.40 - """, { - - 'subjects': [ - {'value': '25040.40', - 'scheme': 'ICS'} - ] - }) + """, + {"subjects": [{"value": "25040.40", "scheme": "ICS"}]}, + ) check_transformation( """ @@ -158,8 +144,8 @@ def test_subject_classification(app): PACS 25.80.Nv - """, { - } + """, + {}, ) check_transformation( """ @@ -168,7 +154,7 @@ def test_subject_classification(app): CERN-2018-003-CP """, - {} + {}, ) @@ -180,10 +166,11 @@ def test_created_by_email(app): karolina.przerwa@cern.ch - """, { - 'created_by': {'_email': 'karolina.przerwa@cern.ch', - 'type': 'user'}, - }) + """, + { + "created_by": {"_email": "karolina.przerwa@cern.ch", "type": "user"}, + }, + ) check_transformation( """ @@ -193,11 +180,12 @@ def test_created_by_email(app): karolina.przerwa@cern.ch - """, { - 'created_by': {'type': 'user', - '_email': 'karolina.przerwa@cern.ch'}, - '_created': '2018-07-16', - }) + """, + { + "created_by": {"type": "user", "_email": "karolina.przerwa@cern.ch"}, + "_created": "2018-07-16", + }, + ) def test_created(app): @@ -208,10 +196,9 @@ def test_created(app): SPR201701 - """, { - 'source': 'SPR', - '_created': '2017-01-01' - }) + """, + {"source": "SPR", "_created": "2017-01-01"}, + ) check_transformation( """ @@ -220,11 +207,13 @@ def test_created(app): random text - """, { - 'internal_notes': [ - {'value': 'random text'}, + """, + { + "internal_notes": [ + {"value": "random text"}, ] - }) + }, + ) check_transformation( """ @@ -237,12 +226,13 @@ def test_created(app): karolina.przerwa@cern.ch - """, { - 'created_by': {'type': 'user', - '_email': 'karolina.przerwa@cern.ch'}, - 'source': 'SPR', - '_created': '2017-01-01', - }) + """, + { + "created_by": {"type": "user", "_email": "karolina.przerwa@cern.ch"}, + "source": "SPR", + "_created": "2017-01-01", + }, + ) def test_collections(app): @@ -253,37 +243,43 @@ def test_collections(app): LEGSERLIB - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'tags': ['LEGSERLIB'], - 'has_tags': True, + "tags": ["LEGSERLIB"], + "has_tags": True, }, - }) + }, + ) check_transformation( """ LEGSERLIB - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'tags': ['LEGSERLIB'], - 'has_tags': True, + "tags": ["LEGSERLIB"], + "has_tags": True, }, - }) + }, + ) check_transformation( """ LEGSERLIB - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'tags': ['LEGSERLIB'], - 'has_tags': True, + "tags": ["LEGSERLIB"], + "has_tags": True, }, - }) + }, + ) check_transformation( """ @@ -291,12 +287,12 @@ def test_collections(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'tags': ['LEGSERLIBINTLAW'], - 'has_tags': True, + "tags": ["LEGSERLIBINTLAW"], + "has_tags": True, }, - } + }, ) check_transformation( """ @@ -305,12 +301,12 @@ def test_collections(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'tags': ['BOOKSHOP'], - 'has_tags': True, + "tags": ["BOOKSHOP"], + "has_tags": True, }, - } + }, ) check_transformation( """ @@ -319,12 +315,12 @@ def test_collections(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'tags': ['BOOKSHOP'], - 'has_tags': True, + "tags": ["BOOKSHOP"], + "has_tags": True, }, - } + }, ) check_transformation( """ @@ -333,12 +329,12 @@ def test_collections(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'tags': ['LEGSERLIBLEGRES'], - 'has_tags': True, + "tags": ["LEGSERLIBLEGRES"], + "has_tags": True, }, - } + }, ) @@ -350,33 +346,41 @@ def test_document_type(app): BOOK - """, { - 'document_type': 'BOOK', - }) + """, + { + "document_type": "BOOK", + }, + ) check_transformation( """ 21 - """, { - 'document_type': 'BOOK', - }) + """, + { + "document_type": "BOOK", + }, + ) check_transformation( """ 42 - """, { - 'document_type': 'PROCEEDINGS', - }) + """, + { + "document_type": "PROCEEDINGS", + }, + ) check_transformation( """ 43 - """, { - 'document_type': 'PROCEEDINGS', - }) + """, + { + "document_type": "PROCEEDINGS", + }, + ) check_transformation( """ @@ -386,7 +390,7 @@ def test_document_type(app): REPORT """, - {'document_type': 'BOOK'} + {"document_type": "BOOK"}, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -395,7 +399,7 @@ def test_document_type(app): virTScvyb """, - {'document_type': 'BOOK'} + {"document_type": "BOOK"}, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -407,7 +411,7 @@ def test_document_type(app): 21 """, - {'document_type': 'BOOK'} + {"document_type": "BOOK"}, ) @@ -422,14 +426,16 @@ def test_document_type_collection(app): BOOK - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'tags': ['LEGSERLIB'], - 'has_tags': True, + "tags": ["LEGSERLIB"], + "has_tags": True, }, - 'document_type': 'BOOK', - }) + "document_type": "BOOK", + }, + ) check_transformation( """ @@ -438,14 +444,16 @@ def test_document_type_collection(app): BOOK - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'tags': ['LEGSERLIB'], - 'has_tags': True, + "tags": ["LEGSERLIB"], + "has_tags": True, }, - 'document_type': 'BOOK', - }) + "document_type": "BOOK", + }, + ) def test_urls(app): @@ -463,32 +471,36 @@ def test_urls(app): Description - """, { - "_migration": - { - **get_migration_dict(), - 'eitems_has_files': True, - 'eitems_file_links': [ - {'description': 'Description', - 'value': 'http://cds.cern.ch/record/1393420/files/NF-EN-13480-2-AC6.pdf'}] - } - } + """, + { + "_migration": { + **get_migration_dict(), + "eitems_has_files": True, + "eitems_file_links": [ + { + "description": "Description", + "value": "http://cds.cern.ch/record/1393420/files/NF-EN-13480-2-AC6.pdf", + } + ], + } + }, ) check_transformation( """ https://cds.cern.ch/record/12345/files/abc.pdf - """, { - "_migration": - { - **get_migration_dict(), - 'eitems_has_files': True, - 'eitems_file_links': [ - { - 'value': 'https://cds.cern.ch/record/12345/files/abc.pdf'}] - } - }) + """, + { + "_migration": { + **get_migration_dict(), + "eitems_has_files": True, + "eitems_file_links": [ + {"value": "https://cds.cern.ch/record/12345/files/abc.pdf"} + ], + } + }, + ) check_transformation( """ @@ -502,12 +514,14 @@ def test_urls(app): http://another.domain/with/a/path pdfa - """, { - 'urls': [ - {'value': 'http://awesome.domain/with/a/path'}, - {'value': 'http://another.domain/with/a/path'}, + """, + { + "urls": [ + {"value": "http://awesome.domain/with/a/path"}, + {"value": "http://another.domain/with/a/path"}, ], - }) + }, + ) check_transformation( """ @@ -516,15 +530,14 @@ def test_urls(app): """, { - "_migration": - { - **get_migration_dict(), - 'eitems_has_ebl': True, - 'eitems_ebl': [ - { - 'value': 'https://cdsweb.cern.ch/auth.py?r=EBLIB_P_1139560'}] - } - } + "_migration": { + **get_migration_dict(), + "eitems_has_ebl": True, + "eitems_ebl": [ + {"value": "https://cdsweb.cern.ch/auth.py?r=EBLIB_P_1139560"} + ], + } + }, ) check_transformation( """ @@ -534,15 +547,16 @@ def test_urls(app): """, { - "_migration": - { - **get_migration_dict(), - 'eitems_has_external': True, - 'eitems_external': [ - { - 'value': 'https://learning.oreilly.com/library/view/-/9781118491300/?ar'}] - } - } + "_migration": { + **get_migration_dict(), + "eitems_has_external": True, + "eitems_external": [ + { + "value": "https://learning.oreilly.com/library/view/-/9781118491300/?ar" + } + ], + } + }, ) check_transformation( """ @@ -552,15 +566,14 @@ def test_urls(app): """, { - "_migration": - { - **get_migration_dict(), - 'eitems_has_proxy': True, - 'eitems_proxy': [ - { - 'value': 'https://www.worldscientific.com/toc/rast/10'}] - } - } + "_migration": { + **get_migration_dict(), + "eitems_has_proxy": True, + "eitems_proxy": [ + {"value": "https://www.worldscientific.com/toc/rast/10"} + ], + } + }, ) check_transformation( """ @@ -574,21 +587,20 @@ def test_urls(app): """, { - "_migration": - { - **get_migration_dict(), - 'eitems_has_ebl': True, - 'eitems_ebl': [ - { - 'value': 'https://cdsweb.cern.ch/auth.py?r=EBLIB_P_1139560'}, - ], - 'eitems_external': [ - { - 'value': 'https://learning.oreilly.com/library/view/-/9781118491300/?ar'}, - ], - 'eitems_has_external': True, - } - } + "_migration": { + **get_migration_dict(), + "eitems_has_ebl": True, + "eitems_ebl": [ + {"value": "https://cdsweb.cern.ch/auth.py?r=EBLIB_P_1139560"}, + ], + "eitems_external": [ + { + "value": "https://learning.oreilly.com/library/view/-/9781118491300/?ar" + }, + ], + "eitems_has_external": True, + } + }, ) check_transformation( """ @@ -596,10 +608,11 @@ def test_urls(app): google.com description - """, { - 'urls': [{'value': 'google.com', - 'description': 'description'}], - }) + """, + { + "urls": [{"value": "google.com", "description": "description"}], + }, + ) def test_authors(app): @@ -641,49 +654,39 @@ def test_authors(app): CERN Univ. Gent - """, { - 'authors': [ - { - 'full_name': 'Frampton, Paul H', - 'roles': ['EDITOR'], - 'alternative_names': 'Neubert, Matthias' - }, - { - 'full_name': 'Glashow, Sheldon Lee', - 'roles': ['EDITOR'] - }, + """, + { + "authors": [ { - 'full_name': 'Van Dam, Hendrik', - 'roles': ['EDITOR'] + "full_name": "Frampton, Paul H", + "roles": ["EDITOR"], + "alternative_names": "Neubert, Matthias", }, + {"full_name": "Glashow, Sheldon Lee", "roles": ["EDITOR"]}, + {"full_name": "Van Dam, Hendrik", "roles": ["EDITOR"]}, { - 'full_name': 'Seyfert, Paul', - 'roles': ['AUTHOR'], - 'affiliations': [{'name': 'CERN'}], - 'identifiers': [ - {'scheme': 'INSPIRE ID', - 'value': 'INSPIRE-00341737'}, - {'scheme': 'CERN', - 'value': '692828'}, - {'scheme': 'CDS', - 'value': '2079441'} + "full_name": "Seyfert, Paul", + "roles": ["AUTHOR"], + "affiliations": [{"name": "CERN"}], + "identifiers": [ + {"scheme": "INSPIRE ID", "value": "INSPIRE-00341737"}, + {"scheme": "CERN", "value": "692828"}, + {"scheme": "CDS", "value": "2079441"}, ], - }, { - 'full_name': 'John Doe', - 'roles': ['AUTHOR'], - 'affiliations': [{'name': 'CERN'}, - {'name': 'Univ. Gent'}], + "full_name": "John Doe", + "roles": ["AUTHOR"], + "affiliations": [{"name": "CERN"}, {"name": "Univ. Gent"}], }, { - 'full_name': 'Jane Doe', - 'roles': ['AUTHOR'], - 'affiliations': [{'name': 'CERN'}, - {'name': 'Univ. Gent'}], - } + "full_name": "Jane Doe", + "roles": ["AUTHOR"], + "affiliations": [{"name": "CERN"}, {"name": "Univ. Gent"}], + }, ], - }) + }, + ) check_transformation( """ @@ -698,18 +701,15 @@ def test_authors(app): """, { - 'authors': [ - { - 'full_name': 'Frampton, Paul H', - 'roles': ['EDITOR'], - }, + "authors": [ { - 'full_name': 'Glashow, Sheldon Lee', - 'roles': ['EDITOR'] + "full_name": "Frampton, Paul H", + "roles": ["EDITOR"], }, + {"full_name": "Glashow, Sheldon Lee", "roles": ["EDITOR"]}, ], - 'other_authors': True, - } + "other_authors": True, + }, ) with pytest.raises(UnexpectedValue): @@ -723,7 +723,7 @@ def test_authors(app): et al. """, - {} + {}, ) @@ -736,19 +736,23 @@ def test_corporate_author(app): Springer - """, { - 'authors': [{'full_name': 'Springer', 'type': 'ORGANISATION'}], - }) + """, + { + "authors": [{"full_name": "Springer", "type": "ORGANISATION"}], + }, + ) check_transformation( """ Marston, R M - """, { - 'authors': [ - {'full_name': 'Marston, R M', 'type': 'ORGANISATION'}, + """, + { + "authors": [ + {"full_name": "Marston, R M", "type": "ORGANISATION"}, ], - }) + }, + ) def test_collaborations(app): @@ -763,8 +767,12 @@ def test_collaborations(app): ATLAS Collaboration """, - {'authors': [{'full_name': 'PH-EP', 'type': 'ORGANISATION'}, - {'full_name': 'ATLAS', 'type': 'ORGANISATION'}]} + { + "authors": [ + {"full_name": "PH-EP", "type": "ORGANISATION"}, + {"full_name": "ATLAS", "type": "ORGANISATION"}, + ] + }, ) @@ -786,16 +794,18 @@ def test_publication_info(app): """, { - 'publication_info': [{ - 'page_start': 1692, - 'page_end': 1695, - 'year': 2007, - 'journal_title': 'Radiat. Meas.', - 'journal_issue': '10', - 'journal_volume': '42', - 'material': 'BOOK', - }], - } + "publication_info": [ + { + "page_start": 1692, + "page_end": 1695, + "year": 2007, + "journal_title": "Radiat. Meas.", + "journal_issue": "10", + "journal_volume": "42", + "material": "BOOK", + } + ], + }, ) check_transformation( """ @@ -811,20 +821,20 @@ def test_publication_info(app): """, { - 'publication_info': [{ - 'page_start': 1692, - 'page_end': 1695, - 'year': 2007, - 'journal_title': 'Radiat. Meas.', - 'journal_issue': '10', - 'journal_volume': '42', - }], - 'conference_info': { - 'identifiers': [ - {'scheme': 'CERN_CODE', 'value': 'fsihfifri'} - ] - } - } + "publication_info": [ + { + "page_start": 1692, + "page_end": 1695, + "year": 2007, + "journal_title": "Radiat. Meas.", + "journal_issue": "10", + "journal_volume": "42", + } + ], + "conference_info": { + "identifiers": [{"scheme": "CERN_CODE", "value": "fsihfifri"}] + }, + }, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -838,15 +848,17 @@ def test_publication_info(app): """, { - 'publication_info': [{ - 'page_start': 1692, - 'page_end': 1695, - 'year': 2007, - 'journal_title': 'Radiat. Meas.', - 'journal_issue': '10', - 'journal_volume': '42', - }] - } + "publication_info": [ + { + "page_start": 1692, + "page_end": 1695, + "year": 2007, + "journal_title": "Radiat. Meas.", + "journal_issue": "10", + "journal_volume": "42", + } + ] + }, ) check_transformation( """ @@ -855,11 +867,7 @@ def test_publication_info(app): Random text """, - { - 'publication_info': [ - {'note': '1692 numebrs text etc Random text'} - ] - } + {"publication_info": [{"note": "1692 numebrs text etc Random text"}]}, ) check_transformation( """ @@ -869,22 +877,22 @@ def test_publication_info(app): 1 """, - {'document_type': 'PERIODICAL_ISSUE', - '_migration': { - **get_migration_dict(), - 'journal_record_legacy_recid': "2155631", - 'has_journal': True, - }, - 'conference_info': { - 'identifiers': [ - {'scheme': 'CERN_CODE', 'value': 'genoa20160330'} - ] - }, - 'publication_info': [ - {'page_start': 1, - } - ] - } + { + "document_type": "PERIODICAL_ISSUE", + "_migration": { + **get_migration_dict(), + "journal_record_legacy_recid": "2155631", + "has_journal": True, + }, + "conference_info": { + "identifiers": [{"scheme": "CERN_CODE", "value": "genoa20160330"}] + }, + "publication_info": [ + { + "page_start": 1, + } + ], + }, ) check_transformation( """ @@ -898,15 +906,17 @@ def test_publication_info(app): """, { - 'publication_info': [{ - 'page_start': 1692, - 'page_end': 1695, - 'year': 2007, - 'journal_title': 'Radiat. Meas.', - 'journal_issue': '10', - 'note': '1692 numebrs text etc Random text', - }] - } + "publication_info": [ + { + "page_start": 1692, + "page_end": 1695, + "year": 2007, + "journal_title": "Radiat. Meas.", + "journal_issue": "10", + "note": "1692 numebrs text etc Random text", + } + ] + }, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -921,16 +931,17 @@ def test_publication_info(app): """, { - 'publication_info': [{ - 'page_start': 1692, - 'page_end': 1695, - 'year': 2007, - 'journal_title': 'Radiat. Meas.', - 'journal_issue': '10', - 'pubinfo_freetext': '1692 numebrs ' - 'text etc Random text', - }] - } + "publication_info": [ + { + "page_start": 1692, + "page_end": 1695, + "year": 2007, + "journal_title": "Radiat. Meas.", + "journal_issue": "10", + "pubinfo_freetext": "1692 numebrs " "text etc Random text", + } + ] + }, ) @@ -946,12 +957,12 @@ def test_extensions(app): """, { - 'extensions': { - 'standard_review:applicability': 'applicable at CERN', - 'standard_review:checkdate': 'Reviewed in December 2019', - 'standard_review:expert': 'Expert ICS-25.160', + "extensions": { + "standard_review:applicability": "applicable at CERN", + "standard_review:checkdate": "Reviewed in December 2019", + "standard_review:expert": "Expert ICS-25.160", }, - } + }, ) check_transformation( """ @@ -963,13 +974,13 @@ def test_extensions(app): """, { - 'extensions': { - 'standard_review:applicability': 'no longer applicable', - 'standard_review:validity': 'withdrawn', - 'standard_review:checkdate': 'Reviewed in December 2019', - 'standard_review:expert': 'Expert ICS-25.160', + "extensions": { + "standard_review:applicability": "no longer applicable", + "standard_review:validity": "withdrawn", + "standard_review:checkdate": "Reviewed in December 2019", + "standard_review:expert": "Expert ICS-25.160", }, - } + }, ) check_transformation( """ @@ -990,17 +1001,16 @@ def test_extensions(app): """, { - 'extensions': - {'unit:accelerator': ['CERN LHC'], - 'unit:experiment': ['ATLAS', 'CMS'], - 'unit:project': ['FCC'], - 'standard_review:applicability': 'no longer applicable', - 'standard_review:validity': 'withdrawn', - 'standard_review:checkdate': 'Reviewed in December 2019', - 'standard_review:expert': 'Expert ICS-25.160', - } - - } + "extensions": { + "unit:accelerator": ["CERN LHC"], + "unit:experiment": ["ATLAS", "CMS"], + "unit:project": ["FCC"], + "standard_review:applicability": "no longer applicable", + "standard_review:validity": "withdrawn", + "standard_review:checkdate": "Reviewed in December 2019", + "standard_review:expert": "Expert ICS-25.160", + } + }, ) @@ -1016,15 +1026,14 @@ def test_related_record(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'has_related': True, - 'related': [{ - 'related_recid': '748392', - "relation_type": "Test text" - }], + "has_related": True, + "related": [ + {"related_recid": "748392", "relation_type": "Test text"} + ], }, - } + }, ) with pytest.raises(ManualMigrationRequired): check_transformation( @@ -1035,13 +1044,14 @@ def test_related_record(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'has_related': True, - 'related': [{'related_recid': '7483924', - 'relation_type': "other"}], + "has_related": True, + "related": [ + {"related_recid": "7483924", "relation_type": "other"} + ], }, - } + }, ) check_transformation( """ @@ -1053,14 +1063,15 @@ def test_related_record(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'has_related': True, - 'related': [ - {'related_recid': '7483924', 'relation_type': 'other'}, - {'related_recid': '748', 'relation_type': 'other'}], + "has_related": True, + "related": [ + {"related_recid": "7483924", "relation_type": "other"}, + {"related_recid": "748", "relation_type": "other"}, + ], }, - } + }, ) @@ -1080,13 +1091,12 @@ def test_accelerator_experiments(app): """, { - 'extensions': - {'unit:accelerator': ['CERN LHC'], - 'unit:experiment': ['ATLAS', 'CMS'], - 'unit:project': ['FCC'], - } - - } + "extensions": { + "unit:accelerator": ["CERN LHC"], + "unit:experiment": ["ATLAS", "CMS"], + "unit:project": ["FCC"], + } + }, ) @@ -1105,17 +1115,22 @@ def test_isbns(app): 9781630811051 electronic version - """, { - 'identifiers': [{ - 'value': '9781630814434', - 'medium': 'electronic version', - 'scheme': 'ISBN', - }, { - 'value': '9781630811051', - 'medium': 'electronic version', - 'scheme': 'ISBN', - }], - }) + """, + { + "identifiers": [ + { + "value": "9781630814434", + "medium": "electronic version", + "scheme": "ISBN", + }, + { + "value": "9781630811051", + "medium": "electronic version", + "scheme": "ISBN", + }, + ], + }, + ) check_transformation( """ @@ -1138,15 +1153,22 @@ def test_isbns(app): 9780691090849 """, - {'identifiers': [ - {'value': '0691090858', 'scheme': 'ISBN'}, - {'value': '9780691090856', 'scheme': 'ISBN'}, - {'value': '9781400889167', 'medium': 'electronic version', - 'scheme': 'ISBN'}, - {'value': '9780691090849', 'medium': 'electronic version', - 'scheme': 'ISBN'}, - ], - } + { + "identifiers": [ + {"value": "0691090858", "scheme": "ISBN"}, + {"value": "9780691090856", "scheme": "ISBN"}, + { + "value": "9781400889167", + "medium": "electronic version", + "scheme": "ISBN", + }, + { + "value": "9780691090849", + "medium": "electronic version", + "scheme": "ISBN", + }, + ], + }, ) with pytest.raises(ManualMigrationRequired): check_transformation( @@ -1159,13 +1181,20 @@ def test_isbns(app): electronic version - """, { - 'identifiers': [{ - 'medium': 'electronic version', 'scheme': 'ISBN', - }, { - 'medium': 'electronic version', 'scheme': 'ISBN', - }], - }) + """, + { + "identifiers": [ + { + "medium": "electronic version", + "scheme": "ISBN", + }, + { + "medium": "electronic version", + "scheme": "ISBN", + }, + ], + }, + ) check_transformation( """ @@ -1178,18 +1207,23 @@ def test_isbns(app): 9781630811051 electronic version (v.1) - """, { - 'identifiers': [{ - 'value': '9781630814434', - 'medium': 'electronic version', - 'scheme': 'ISBN', - }, { - 'value': '9781630811051', - 'medium': 'electronic version', - 'scheme': 'ISBN', - }], - 'volume': '(v.1)', - }) + """, + { + "identifiers": [ + { + "value": "9781630814434", + "medium": "electronic version", + "scheme": "ISBN", + }, + { + "value": "9781630811051", + "medium": "electronic version", + "scheme": "ISBN", + }, + ], + "volume": "(v.1)", + }, + ) check_transformation( """ @@ -1203,18 +1237,23 @@ def test_isbns(app): 9781630811051 electronic version (v.1) - """, { - 'identifiers': [{ - 'value': '9781630814434', - 'description': 'description', - 'scheme': 'ISBN', - }, { - 'value': '9781630811051', - 'medium': 'electronic version', - 'scheme': 'ISBN', - }], - 'volume': '(v.1)', - }) + """, + { + "identifiers": [ + { + "value": "9781630814434", + "description": "description", + "scheme": "ISBN", + }, + { + "value": "9781630811051", + "medium": "electronic version", + "scheme": "ISBN", + }, + ], + "volume": "(v.1)", + }, + ) with pytest.raises(ManualMigrationRequired): check_transformation( @@ -1229,18 +1268,23 @@ def test_isbns(app): 9781630811051 electronic version (v.1) - """, { - 'identifiers': [{ - 'value': '9781630814434', - 'medium': 'electronic version', - 'scheme': 'ISBN', - }, { - 'value': '9781630811051', - 'medium': 'electronic version', - 'scheme': 'ISBN', - }], - 'volume': '(v.1)', - }) + """, + { + "identifiers": [ + { + "value": "9781630814434", + "medium": "electronic version", + "scheme": "ISBN", + }, + { + "value": "9781630811051", + "medium": "electronic version", + "scheme": "ISBN", + }, + ], + "volume": "(v.1)", + }, + ) def test_report_numbers(app): @@ -1252,22 +1296,23 @@ def test_report_numbers(app): arXiv arXiv:1808.02335 - """, { - 'alternative_identifiers': [{ - 'value': 'arXiv:1808.02335', - 'scheme': 'arXiv' - }], - }) + """, + { + "alternative_identifiers": [ + {"value": "arXiv:1808.02335", "scheme": "arXiv"} + ], + }, + ) check_transformation( """ hep-th/9509119 - """, { - 'identifiers': [{ - 'value': 'hep-th/9509119', 'scheme': 'REPORT_NUMBER' - }], - }) + """, + { + "identifiers": [{"value": "hep-th/9509119", "scheme": "REPORT_NUMBER"}], + }, + ) check_transformation( """ @@ -1275,50 +1320,70 @@ def test_report_numbers(app): arXiv:1808.02335 hep-ex - """, { - 'alternative_identifiers': [{ - 'value': 'arXiv:1808.02335', - 'scheme': 'arXiv', - }], - 'subjects': [{ - 'scheme': 'arXiv', - 'value': 'hep-ex', - }], - }) + """, + { + "alternative_identifiers": [ + { + "value": "arXiv:1808.02335", + "scheme": "arXiv", + } + ], + "subjects": [ + { + "scheme": "arXiv", + "value": "hep-ex", + } + ], + }, + ) check_transformation( """ arXiv arXiv:1808.02335 - """, { - 'alternative_identifiers': [{ - 'value': 'arXiv:1808.02335', - 'scheme': 'arXiv', - }], - }) + """, + { + "alternative_identifiers": [ + { + "value": "arXiv:1808.02335", + "scheme": "arXiv", + } + ], + }, + ) check_transformation( """ CERN-THESIS-2018-004 - """, { - 'identifiers': [{ - 'value': 'CERN-THESIS-2018-004', 'hidden': True, - 'scheme': 'REPORT_NUMBER' - }], - }) + """, + { + "identifiers": [ + { + "value": "CERN-THESIS-2018-004", + "hidden": True, + "scheme": "REPORT_NUMBER", + } + ], + }, + ) check_transformation( """ CERN-ISOLDE-2018-001 - """, { - 'identifiers': [{ - 'value': 'CERN-ISOLDE-2018-001', 'hidden': True, - 'scheme': 'REPORT_NUMBER' - }], - }) + """, + { + "identifiers": [ + { + "value": "CERN-ISOLDE-2018-001", + "hidden": True, + "scheme": "REPORT_NUMBER", + } + ], + }, + ) check_transformation( """ @@ -1330,28 +1395,36 @@ def test_report_numbers(app): ATL-COM-PHYS-2017 - """, { - 'identifiers': [ - {'value': 'NAPAC-2016-MOPOB23', - 'scheme': 'REPORT_NUMBER'}, - {'value': 'ATL-COM-PHYS-2018-980', 'hidden': True, - 'scheme': 'REPORT_NUMBER'}, - {'value': 'ATL-COM-PHYS-2017', 'hidden': True, - 'scheme': 'REPORT_NUMBER'}, + """, + { + "identifiers": [ + {"value": "NAPAC-2016-MOPOB23", "scheme": "REPORT_NUMBER"}, + { + "value": "ATL-COM-PHYS-2018-980", + "hidden": True, + "scheme": "REPORT_NUMBER", + }, + { + "value": "ATL-COM-PHYS-2017", + "hidden": True, + "scheme": "REPORT_NUMBER", + }, ], - }) + }, + ) with pytest.raises(MissingRequiredField): check_transformation( """ hep-th/9509119 - """, { - 'identifiers': [{ - 'value': 'hep-th/9509119', - 'scheme': 'REPORT_NUMBER' - }], - }) + """, + { + "identifiers": [ + {"value": "hep-th/9509119", "scheme": "REPORT_NUMBER"} + ], + }, + ) with pytest.raises(MissingRule): check_transformation( """ @@ -1359,8 +1432,9 @@ def test_report_numbers(app): LANL EDS math-ph - """, { - }) + """, + {}, + ) def test_dois(app): @@ -1374,14 +1448,18 @@ def test_dois(app): data source - """, { - 'identifiers': [{ - 'source': 'source', - 'material': 'data', - 'value': '10.1007/978-1-4613-0247-6', - 'scheme': 'DOI', - }], - }) + """, + { + "identifiers": [ + { + "source": "source", + "material": "data", + "value": "10.1007/978-1-4613-0247-6", + "scheme": "DOI", + } + ], + }, + ) def test_alternative_identifiers(app): @@ -1393,12 +1471,16 @@ def test_alternative_identifiers(app): EBL 5231528 - """, { - 'alternative_identifiers': [{ - 'scheme': 'EBL', - 'value': '5231528', - }], - }) + """, + { + "alternative_identifiers": [ + { + "scheme": "EBL", + "value": "5231528", + } + ], + }, + ) check_transformation( """ @@ -1410,16 +1492,19 @@ def test_alternative_identifiers(app): Inspire 2365039 - """, { - 'conference_info': { - 'identifiers': [ - {'scheme': 'INSPIRE_CNUM', 'value': '2365039'}], + """, + { + "conference_info": { + "identifiers": [{"scheme": "INSPIRE_CNUM", "value": "2365039"}], }, - 'alternative_identifiers': [{ - 'scheme': 'Inspire', - 'value': '2365039', - }], - }) + "alternative_identifiers": [ + { + "scheme": "Inspire", + "value": "2365039", + } + ], + }, + ) with pytest.raises(UnexpectedValue): check_transformation( @@ -1428,8 +1513,9 @@ def test_alternative_identifiers(app): Random 2365039 - """, { - }) + """, + {}, + ) check_transformation( """ @@ -1437,8 +1523,9 @@ def test_alternative_identifiers(app): CERCER 2365039 - """, { - }) + """, + {}, + ) check_transformation( """ @@ -1446,8 +1533,9 @@ def test_alternative_identifiers(app): SLAC 5231528 - """, { - }) + """, + {}, + ) check_transformation( """ @@ -1460,14 +1548,16 @@ def test_alternative_identifiers(app): EBL 5231528 - """, { - 'alternative_identifiers': [ + """, + { + "alternative_identifiers": [ { - 'value': '5231528', - 'scheme': 'EBL', + "value": "5231528", + "scheme": "EBL", } ] - }) + }, + ) check_transformation( """ @@ -1485,29 +1575,36 @@ def test_alternative_identifiers(app): 10.1103/PhysRevLett.121.052004 publication - """, { - 'identifiers': [ - {'value': '10.1007/s00269-016-0862-1', 'scheme': 'DOI'}, - {'value': '10.1103/PhysRevLett.121.052004', - 'scheme': 'DOI'}, - {'value': '10.1103/PhysRevLett.121.052004', - 'scheme': 'DOI', - 'material': 'publication', - 'source': 'arXiv'} + """, + { + "identifiers": [ + {"value": "10.1007/s00269-016-0862-1", "scheme": "DOI"}, + {"value": "10.1103/PhysRevLett.121.052004", "scheme": "DOI"}, + { + "value": "10.1103/PhysRevLett.121.052004", + "scheme": "DOI", + "material": "publication", + "source": "arXiv", + }, ], - }) + }, + ) check_transformation( """ DLC 92074207 - """, { - 'alternative_identifiers': [{ - 'scheme': 'DLC', - 'value': '92074207', - }], - }) + """, + { + "alternative_identifiers": [ + { + "scheme": "DLC", + "value": "92074207", + } + ], + }, + ) # ignore 035__9== arXiv check_transformation( """ @@ -1515,10 +1612,9 @@ def test_alternative_identifiers(app): arXiv 5231528 - """, { - 'alternative_identifiers': [ - {'scheme': 'arXiv', 'value': '5231528'}] - }) + """, + {"alternative_identifiers": [{"scheme": "arXiv", "value": "5231528"}]}, + ) def test_arxiv_eprints(app): @@ -1531,16 +1627,17 @@ def test_arxiv_eprints(app): arXiv:1209.5665 math-ph - """, { - 'alternative_identifiers': [{ - 'scheme': 'arXiv', - 'value': 'arXiv:1209.5665', - }], - 'subjects': [{ - 'scheme': 'arXiv', - 'value': 'math-ph' - }] - }) + """, + { + "alternative_identifiers": [ + { + "scheme": "arXiv", + "value": "arXiv:1209.5665", + } + ], + "subjects": [{"scheme": "arXiv", "value": "math-ph"}], + }, + ) check_transformation( """ @@ -1553,16 +1650,17 @@ def test_arxiv_eprints(app): arXiv:1209.5665 math.GT - """, { - 'alternative_identifiers': [{ - 'value': 'arXiv:1209.5665', - 'scheme': 'arXiv' - }], - 'subjects': [ - {'scheme': 'arXiv', 'value': 'math-ph'}, - {'scheme': 'arXiv', 'value': 'math.GT'}, - ] - }) + """, + { + "alternative_identifiers": [ + {"value": "arXiv:1209.5665", "scheme": "arXiv"} + ], + "subjects": [ + {"scheme": "arXiv", "value": "math-ph"}, + {"scheme": "arXiv", "value": "math.GT"}, + ], + }, + ) with pytest.raises(UnexpectedValue): check_transformation( """ @@ -1571,12 +1669,16 @@ def test_arxiv_eprints(app): arXiv:1209.5665 math-phss - """, { - 'arxiv_eprints': [{ - 'categories': ['math-ph'], - 'value': 'arXiv:1209.5665', - }], - }) + """, + { + "arxiv_eprints": [ + { + "categories": ["math-ph"], + "value": "arXiv:1209.5665", + } + ], + }, + ) def test_languages(app): @@ -1587,50 +1689,62 @@ def test_languages(app): eng - """, { - 'languages': ['EN'], - }) + """, + { + "languages": ["EN"], + }, + ) check_transformation( """ english - """, { - 'languages': ['EN'], - }) + """, + { + "languages": ["EN"], + }, + ) check_transformation( """ fre - """, { - 'languages': ['FR'], - }) + """, + { + "languages": ["FR"], + }, + ) check_transformation( """ pl - """, { - 'languages': ['PL'], - }) + """, + { + "languages": ["PL"], + }, + ) check_transformation( """ ger - """, { - 'languages': ['DE'], - }) + """, + { + "languages": ["DE"], + }, + ) with pytest.raises(UnexpectedValue): check_transformation( """ xxxxxxxx - """, { - 'languages': ['DE'], - }) + """, + { + "languages": ["DE"], + }, + ) def test_editions(app): @@ -1641,9 +1755,9 @@ def test_editions(app): 3rd ed. - """, { - 'edition': '3rd' - }) + """, + {"edition": "3rd"}, + ) def test_imprint(app): @@ -1657,15 +1771,17 @@ def test_imprint(app): 2013 2015 - """, { - 'publication_year': "2013", - 'imprint': { - 'place': 'Sydney', - 'publisher': 'Allen & Unwin', - 'date': '2013', - 'reprint_date': '2015', + """, + { + "publication_year": "2013", + "imprint": { + "place": "Sydney", + "publisher": "Allen & Unwin", + "date": "2013", + "reprint_date": "2015", }, - }) + }, + ) @pytest.mark.skip @@ -1679,18 +1795,20 @@ def test_preprint_date(app): CERN 19 Jan 2016 - """, { - 'preprint_date': '2016-01-19', - }) + """, + { + "preprint_date": "2016-01-19", + }, + ) check_transformation( """ Geneva CERN - """, { - - }) + """, + {}, + ) with pytest.raises(ManualMigrationRequired): check_transformation( """ @@ -1699,9 +1817,11 @@ def test_preprint_date(app): CERN 33 Jan 2016 - """, { - 'preprint_date': '2016-01-19', - }) + """, + { + "preprint_date": "2016-01-19", + }, + ) def test_number_of_pages(app): @@ -1712,41 +1832,46 @@ def test_number_of_pages(app): 373 p - """, { - 'number_of_pages': "373", - }) + """, + { + "number_of_pages": "373", + }, + ) check_transformation( """ 480 p. ; 1 CD-ROM suppl - """, { - 'number_of_pages': "480", - 'physical_copy_description': '1 CD-ROM' - }) + """, + {"number_of_pages": "480", "physical_copy_description": "1 CD-ROM"}, + ) check_transformation( """ 42 p. ; 2 CD-ROM ; 1 DVD, 1 vhs - """, { - 'number_of_pages': "42", - 'physical_copy_description': '2 CD-ROM, 1 DVD, 1 VHS' - }) + """, + { + "number_of_pages": "42", + "physical_copy_description": "2 CD-ROM, 1 DVD, 1 VHS", + }, + ) check_transformation( """ - """, { - }) + """, + {}, + ) check_transformation( """ mult. p - """, { - }) + """, + {}, + ) with pytest.raises(UnexpectedValue): check_transformation( @@ -1754,15 +1879,17 @@ def test_number_of_pages(app): 2 v - """, { - }) + """, + {}, + ) check_transformation( """ 42 p. + 17 p - """, { - }) + """, + {}, + ) check_transformation( """ @@ -1770,8 +1897,9 @@ def test_number_of_pages(app): amendment A1 (18 p) + amendment A2 (18 p) - """, { - }) + """, + {}, + ) check_transformation( """ @@ -1779,15 +1907,17 @@ def test_number_of_pages(app): amendment A1 (18 p) + amendment A2 (18 p) - """, { - }) + """, + {}, + ) check_transformation( """ 42 p. ; E22 - """, { - }) + """, + {}, + ) def test_abstracts(app): @@ -1798,9 +1928,11 @@ def test_abstracts(app): The publication... - """, { - 'abstract': 'The publication...', - }) + """, + { + "abstract": "The publication...", + }, + ) check_transformation( """ @@ -1810,12 +1942,12 @@ def test_abstracts(app): Does application... - """, { - 'abstract': 'The publication...', - 'alternative_abstracts': [ - 'Does application...' - ], - }) + """, + { + "abstract": "The publication...", + "alternative_abstracts": ["Does application..."], + }, + ) with pytest.raises(MissingRequiredField): check_transformation( """ @@ -1825,11 +1957,9 @@ def test_abstracts(app): Does application... - """, { - 'abstract': - 'Does application...' - - }) + """, + {"abstract": "Does application..."}, + ) @pytest.mark.skip @@ -1847,19 +1977,21 @@ def test_funding_info(app): AIDA-2020 openAccess - """, { - 'funding_info': [ + """, + { + "funding_info": [ { - 'agency': 'CERN Technical Student Program', + "agency": "CERN Technical Student Program", }, { - 'agency': 'FP7', - 'grant_number': '654168', - 'project_number': 'AIDA-2020', - 'openaccess': True, + "agency": "FP7", + "grant_number": "654168", + "project_number": "AIDA-2020", + "openaccess": True, }, ] - }) + }, + ) with pytest.raises(UnexpectedValue): check_transformation( """ @@ -1874,19 +2006,21 @@ def test_funding_info(app): AIDA-2020 openAccedafss - """, { - 'funding_info': [ + """, + { + "funding_info": [ { - 'agency': 'CERN Technical Student Program', + "agency": "CERN Technical Student Program", }, { - 'agency': 'FP7', - 'grant_number': '654168', - 'project_number': 'AIDA-2020', - 'openaccess': True, + "agency": "FP7", + "grant_number": "654168", + "project_number": "AIDA-2020", + "openaccess": True, }, ] - }) + }, + ) def test_license(app): @@ -1910,31 +2044,33 @@ def test_license(app): SCOAP3 DAI/7161287 - """, { - 'licenses': [ + """, + { + "licenses": [ { - 'license': { - 'url': 'http://arxiv.org/licenses/nonexclusive-distrib/1.0/', - 'name': None, + "license": { + "url": "http://arxiv.org/licenses/nonexclusive-distrib/1.0/", + "name": None, } }, { - 'license': { - 'name': 'CC-BY-4.0', - 'url': None, + "license": { + "name": "CC-BY-4.0", + "url": None, }, - 'material': 'preprint', + "material": "preprint", }, { - 'license': { - 'name': 'CC-BY-4.0', - 'url': None, + "license": { + "name": "CC-BY-4.0", + "url": None, }, - 'material': 'publication', - 'internal_note': 'DAI/7161287', - } + "material": "publication", + "internal_note": "DAI/7161287", + }, ] - }) + }, + ) def test_copyright(app): @@ -1959,25 +2095,23 @@ def test_copyright(app): http://creativecommons.org/licenses/by/4.0 - """, { - 'copyrights': [ + """, + { + "copyrights": [ { - 'holder': 'd', - 'statement': 'f', - 'year': 2013, - 'url': 'u', + "holder": "d", + "statement": "f", + "year": 2013, + "url": "u", }, + {"material": "preprint", "holder": "CERN", "year": 2018}, { - 'material': 'preprint', - 'holder': 'CERN', - 'year': 2018 + "statement": "This work is licensed.", + "url": "http://creativecommons.org/licenses/by/4.0", }, - { - 'statement': 'This work is licensed.', - 'url': 'http://creativecommons.org/licenses/by/4.0', - } ] - }) + }, + ) def test_conference_info(app): @@ -2007,19 +2141,22 @@ def test_conference_info(app): SNGHEGE2004 """, - {'conference_info': { - 'identifiers': [ - {'scheme': 'INSPIRE_CNUM', 'value': '1234'}, - {'scheme': 'CERN_CODE', 'value': 'bari20040621'}, - ], - 'title': """2nd Workshop on Science with + { + "conference_info": { + "identifiers": [ + {"scheme": "INSPIRE_CNUM", "value": "1234"}, + {"scheme": "CERN_CODE", "value": "bari20040621"}, + ], + "title": """2nd Workshop on Science with the New Generation of High Energy Gamma-ray Experiments: between Astrophysics and Astroparticle Physics""", - 'place': 'Bari, Italy', - 'dates': '2004-06-21 - 2004-06-21', - 'series': {'number': 2}, - 'country': 'IT', - 'acronym': 'SNGHEGE2004'}} + "place": "Bari, Italy", + "dates": "2004-06-21 - 2004-06-21", + "series": {"number": 2}, + "country": "IT", + "acronym": "SNGHEGE2004", + } + }, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -2040,18 +2177,21 @@ def test_conference_info(app): """, - {'conference_info': { - 'title': """2nd Workshop on Science with the New + { + "conference_info": { + "title": """2nd Workshop on Science with the New Generation of High Energy Gamma-ray Experiments: between Astrophysics and Astroparticle Physics""", - 'place': 'Bari, Italy', - 'identifiers': [ - {'scheme': 'CERN_CODE', 'value': 'bari20040621'}, - ], - 'dates': '2004-06-21 - 2004-06-21', - 'series': {'number': 2}, - 'country_code': 'IT', - 'contact': 'arantza.de.oyanguren.campos@cern.ch'}} + "place": "Bari, Italy", + "identifiers": [ + {"scheme": "CERN_CODE", "value": "bari20040621"}, + ], + "dates": "2004-06-21 - 2004-06-21", + "series": {"number": 2}, + "country_code": "IT", + "contact": "arantza.de.oyanguren.campos@cern.ch", + } + }, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -2078,18 +2218,21 @@ def test_conference_info(app): SNGHEGE2004 """, - {'conference_info': { - 'title': """2nd Workshop on Science with the New + { + "conference_info": { + "title": """2nd Workshop on Science with the New Generation of High Energy Gamma-ray Experiments: between Astrophysics and Astroparticle Physics""", - 'place': 'Bari, Italy', - 'cern_conference_code': 'bari20040621', - 'opening_date': '2004-06-21', - 'series_number': 2, - 'country_code': 'IT', - 'closing_date': '2004-06-21', - 'contact': 'arantza.de.oyanguren.campos@cern.ch', - 'acronym': 'SNGHEGE2004'}} + "place": "Bari, Italy", + "cern_conference_code": "bari20040621", + "opening_date": "2004-06-21", + "series_number": 2, + "country_code": "IT", + "closing_date": "2004-06-21", + "contact": "arantza.de.oyanguren.campos@cern.ch", + "acronym": "SNGHEGE2004", + } + }, ) with pytest.raises(MissingRequiredField): check_transformation( @@ -2098,9 +2241,12 @@ def test_conference_info(app): arantza.de.oyanguren.campos@cern.ch - """, { - 'conference_info': - {'contact': 'arantza.de.oyanguren.campos@cern.ch'}} + """, + { + "conference_info": { + "contact": "arantza.de.oyanguren.campos@cern.ch" + } + }, ) @@ -2116,19 +2262,21 @@ def test_alternative_titles_a(app): Subtitle/LHC """, - {'alternative_titles': [ - {'value': """Study of the impact of stacking on simple + { + "alternative_titles": [ + { + "value": """Study of the impact of stacking on simple hard diffraction events in CMS/LHC""", - 'language': 'EN', - 'type': 'TRANSLATED_TITLE' - }, - { - 'value': 'Subtitle/LHC', - 'language': 'EN', - 'type': 'TRANSLATED_SUBTITLE' - } - ] - } + "language": "EN", + "type": "TRANSLATED_TITLE", + }, + { + "value": "Subtitle/LHC", + "language": "EN", + "type": "TRANSLATED_SUBTITLE", + }, + ] + }, ) @@ -2143,11 +2291,11 @@ def test_title(app): """, { - 'title': 'Incoterms 2010', - 'alternative_titles': [ - {'value': "les règles de l'ICC", 'type': 'SUBTITLE'} - ] - } + "title": "Incoterms 2010", + "alternative_titles": [ + {"value": "les règles de l'ICC", "type": "SUBTITLE"} + ], + }, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -2160,7 +2308,7 @@ def test_title(app): With duplicate title """, - {} + {}, ) @@ -2184,25 +2332,22 @@ def test_alternative_titles(app): """, { - 'alternative_titles': [ - { - 'value': 'Air quality — sampling', - 'type': 'ALTERNATIVE_TITLE', - }, + "alternative_titles": [ { - 'value': u"""part 4: guidance on the metrics""", - 'type': 'SUBTITLE', + "value": "Air quality — sampling", + "type": "ALTERNATIVE_TITLE", }, { - 'value': 'Water quality — sampling', - 'type': 'ALTERNATIVE_TITLE' + "value": """part 4: guidance on the metrics""", + "type": "SUBTITLE", }, + {"value": "Water quality — sampling", "type": "ALTERNATIVE_TITLE"}, { - 'value': u"""part 15: guidance on preservation""", - 'type': 'SUBTITLE' + "value": """part 15: guidance on preservation""", + "type": "SUBTITLE", }, ] - } + }, ) with app.app_context(): check_transformation( @@ -2218,18 +2363,18 @@ def test_alternative_titles(app): """, { - 'document_type': 'BOOK', - 'alternative_titles': [ + "document_type": "BOOK", + "alternative_titles": [ { - 'value': 'Water quality — sampling', - 'type': 'ALTERNATIVE_TITLE', + "value": "Water quality — sampling", + "type": "ALTERNATIVE_TITLE", }, { - 'value': u"""part 15: guidance on the preservation""", - 'type': 'SUBTITLE', + "value": """part 15: guidance on the preservation""", + "type": "SUBTITLE", }, - ] - } + ], + }, ) @@ -2247,9 +2392,7 @@ def test_note(app): No CD-ROM """, - { - 'note': """Translated from ... / No CD-ROM""" - } + {"note": """Translated from ... / No CD-ROM"""}, ) check_transformation( """ @@ -2259,10 +2402,8 @@ def test_note(app): Comments: Book, 380 p., - """, { - 'note': """Comments: Book, 380 p.,""" - - } + """, + {"note": """Comments: Book, 380 p.,"""}, ) @@ -2277,13 +2418,15 @@ def test_table_of_contents(app): """, - {'table_of_content': [ - '2nd Advanced School on Exoplanetary Science: Astrophysics of Exoplanetary Atmospheres', - 'Chapter 1: Modeling Exoplanetary Atmospheres, by Jonathan J. Fortney', - 'Chapter 2: Observational Techniques, by David Sing', - 'Chapter 3: Molecular spectroscopy for Exoplanets by Jonathan Tennyson', - 'Chapter 4: Solar system atmospheres by Davide Grassi.' - ]} + { + "table_of_content": [ + "2nd Advanced School on Exoplanetary Science: Astrophysics of Exoplanetary Atmospheres", + "Chapter 1: Modeling Exoplanetary Atmospheres, by Jonathan J. Fortney", + "Chapter 2: Observational Techniques, by David Sing", + "Chapter 3: Molecular spectroscopy for Exoplanets by Jonathan Tennyson", + "Chapter 4: Solar system atmospheres by Davide Grassi.", + ] + }, ) with pytest.raises(UnexpectedValue): check_transformation( @@ -2293,9 +2436,7 @@ def test_table_of_contents(app): """, - {'table_of_content': [ - - ]} + {"table_of_content": []}, ) @@ -2314,14 +2455,17 @@ def test_standard_numbers(app): BS-EN-ISO-6507-2 """, - {'identifiers': [ - {'value': 'FD-X-60-000', - 'scheme': 'STANDARD_NUMBER'}, - {'value': 'NF-EN-13306', - 'scheme': 'STANDARD_NUMBER'}, - {'value': 'BS-EN-ISO-6507-2', 'hidden': True, - 'scheme': 'STANDARD_NUMBER'}, - ]} + { + "identifiers": [ + {"value": "FD-X-60-000", "scheme": "STANDARD_NUMBER"}, + {"value": "NF-EN-13306", "scheme": "STANDARD_NUMBER"}, + { + "value": "BS-EN-ISO-6507-2", + "hidden": True, + "scheme": "STANDARD_NUMBER", + }, + ] + }, ) with pytest.raises(MissingRequiredField): check_transformation( @@ -2330,14 +2474,17 @@ def test_standard_numbers(app): FD-X-60-000 """, - {'identifiers': [ - {'value': 'FD-X-60-000', - 'scheme': 'STANDARD_NUMBER'}, - {'value': 'NF-EN-13306', - 'scheme': 'STANDARD_NUMBER'}, - {'value': 'BS-EN-ISO-6507-2', 'hidden': True, - 'scheme': 'STANDARD_NUMBER'}, - ]} + { + "identifiers": [ + {"value": "FD-X-60-000", "scheme": "STANDARD_NUMBER"}, + {"value": "NF-EN-13306", "scheme": "STANDARD_NUMBER"}, + { + "value": "BS-EN-ISO-6507-2", + "hidden": True, + "scheme": "STANDARD_NUMBER", + }, + ] + }, ) @@ -2349,19 +2496,14 @@ def test_book_series(app): Minutes - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'serials': [ - { - 'title': 'Minutes', - 'issn': None, - 'volume': None - } - ], - 'has_serial': True, + "serials": [{"title": "Minutes", "issn": None, "volume": None}], + "has_serial": True, } - } + }, ) check_transformation( """ @@ -2371,19 +2513,20 @@ def test_book_series(app): 16 - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'serials': [ + "serials": [ { - 'title': 'De Gruyter studies in mathematical physics', - 'issn': None, - 'volume': '16' + "title": "De Gruyter studies in mathematical physics", + "issn": None, + "volume": "16", } ], - 'has_serial': True, + "has_serial": True, } - } + }, ) check_transformation( """ @@ -2392,19 +2535,20 @@ def test_book_series(app): 267 0081-3869 - """, { - '_migration': { + """, + { + "_migration": { **get_migration_dict(), - 'serials': [ + "serials": [ { - 'title': 'Springer tracts in modern physics', - 'issn': '0081-3869', - 'volume': '267' + "title": "Springer tracts in modern physics", + "issn": "0081-3869", + "volume": "267", } ], - 'has_serial': True, + "has_serial": True, } - } + }, ) @@ -2492,82 +2636,46 @@ def test_541(app): """, { - 'agency_code': "SzGeCERN", + "agency_code": "SzGeCERN", # 'acquisition_source': { # 'datetime': "2019-01-21" # }, - 'creation_date': '2019-01-21', - '_collections': [ - "BOOKSHOP" + "creation_date": "2019-01-21", + "_collections": ["BOOKSHOP"], + "number_of_pages": 401, + "subject_classification": [{"value": "519.226", "schema": "UDC"}], + "languages": ["en"], + "title": "Bayesian networks in fault diagnosis", + "alternative_titles": [ + {"value": "practice and application", "type": "SUBTITLE"} ], - 'number_of_pages': 401, - 'subject_classification': [ - { - 'value': "519.226", - 'schema': "UDC" - } + "legacy_recid": 2654497, + "isbns": [ + {"medium": "print version, hardback", "value": "9789813271487"} ], - 'languages': [ - "en" + "authors": [ + {"role": "editor", "full_name": "Cai, Baoping"}, + {"role": "editor", "full_name": "Liu, Yonghong"}, + {"role": "editor", "full_name": "Hu, Jinqiu"}, + {"role": "editor", "full_name": "Liu, Zengkai"}, + {"role": "editor", "full_name": "Wu, Shengnan"}, + {"role": "editor", "full_name": "Ji, Renjie"}, ], - 'title': "Bayesian networks in fault diagnosis", - 'alternative_titles': [ - {'value': "practice and application", - 'type': 'SUBTITLE'} + "original_source": None, + "external_system_identifiers": [ + {"value": "90.20.00.192.6", "schema": "SCEM"} ], - 'legacy_recid': 2654497, - 'isbns': [ + "$schema": {"$ref": "records/books/book/book-v.0.0.1.json"}, + "document_type": "BOOK", + "imprints": [ { - 'medium': "print version, hardback", - 'value': "9789813271487" + "date": "2019", + "publisher": "World Scientific", + "place": "Singapore", } ], - 'authors': [ - { - 'role': "editor", - 'full_name': "Cai, Baoping" - }, - { - 'role': "editor", - 'full_name': "Liu, Yonghong" - }, - { - 'role': "editor", - 'full_name': "Hu, Jinqiu" - }, - { - 'role': "editor", - 'full_name': "Liu, Zengkai" - }, - { - 'role': "editor", - 'full_name': "Wu, Shengnan" - }, - { - 'role': "editor", - 'full_name': "Ji, Renjie" - } - ], - 'original_source': None, - 'external_system_identifiers': [ - { - 'value': "90.20.00.192.6", - 'schema': "SCEM" - } - ], - '$schema': { - '$ref': "records/books/book/book-v.0.0.1.json" - }, - 'document_type': - "BOOK", - 'imprints': [ - { - 'date': "2019", - 'publisher': "World Scientific", - 'place': "Singapore" - } - ] - }) + }, + ) def test_keywords(app): @@ -2581,10 +2689,10 @@ def test_keywords(app): """, { - 'keywords': [ - {'value': 'Keyword Name 1', 'source': 'PACS'}, + "keywords": [ + {"value": "Keyword Name 1", "source": "PACS"}, ], - } + }, ) @@ -2606,17 +2714,17 @@ def test_volume_barcodes(app): """, dict( - title='Mathematische Methoden der Physik', + title="Mathematische Methoden der Physik", _migration={ **get_migration_dict(), **dict( volumes=[ - dict(barcode='80-1209-8', volume='1'), - dict(barcode='B00004172', volume='1'), + dict(barcode="80-1209-8", volume="1"), + dict(barcode="B00004172", volume="1"), ], - ), } - - ) + ), + }, + ), ) @@ -2641,5 +2749,5 @@ def test_conference_info_multiple_series_number(app): 20150717 """, - dict() + dict(), ) diff --git a/tests/test_cds_dojson.py b/tests/test_cds_dojson.py index 21073cbc..da312296 100644 --- a/tests/test_cds_dojson.py +++ b/tests/test_cds_dojson.py @@ -27,14 +27,17 @@ def test_version(): """Test version import.""" from cds_dojson import __version__ + assert __version__ def test_wrong_model(): """Test cds_marc21 should not be used for `over`.""" from cds_dojson.marc21 import marc21 + with pytest.raises(NotImplementedError): - @marc21.over('test', '...') + + @marc21.over("test", "...") def test(self, key, value): """Testing function.""" - return {'a': 'b'} + return {"a": "b"} diff --git a/tests/test_cli.py b/tests/test_cli.py index c4f74673..1e54da29 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,61 +23,66 @@ # as an Intergovernmental Organization or submit itself to any jurisdiction. """Test cds-dojson CLI and jsonschema compiler.""" -from __future__ import absolute_import - import json import os -import shutil +from importlib.resources import files -import pkg_resources import pytest from click.testing import CliRunner from cds_dojson.cli import compile_schema, convert_yaml2json -@pytest.mark.parametrize('src, compiled', [ - ('records/videos/video/video_src-v1.0.0.json', - 'records/videos/video/video-v1.0.0.json'), - ('records/videos/project/project_src-v1.0.0.json', - 'records/videos/project/project-v1.0.0.json'), - ('deposits/records/videos/video/video_src-v1.0.0.json', - 'deposits/records/videos/video/video-v1.0.0.json'), - ('deposits/records/videos/project/project_src-v1.0.0.json', - 'deposits/records/videos/project/project-v1.0.0.json'), -]) +@pytest.mark.parametrize( + "src, compiled", + [ + ( + "records/videos/video/video_src-v1.0.0.json", + "records/videos/video/video-v1.0.0.json", + ), + ( + "records/videos/project/project_src-v1.0.0.json", + "records/videos/project/project-v1.0.0.json", + ), + ( + "deposits/records/videos/video/video_src-v1.0.0.json", + "deposits/records/videos/video/video-v1.0.0.json", + ), + ( + "deposits/records/videos/project/project_src-v1.0.0.json", + "deposits/records/videos/project/project-v1.0.0.json", + ), + ], +) def test_cli(src, compiled): """Test cds-dojson CLI.""" runner = CliRunner() - result = runner.invoke(compile_schema, [ - pkg_resources.resource_filename('cds_dojson.schemas', src) - ]) + result = runner.invoke(compile_schema, [str(files("cds_dojson.schemas") / src)]) assert 0 == result.exit_code compiled_schema_result = json.loads(result.output) - with open( - pkg_resources.resource_filename('cds_dojson.schemas', - compiled), 'r') as f: + with (files("cds_dojson.schemas") / compiled).open("r") as f: compile_schema_expected = json.load(f) assert compile_schema_expected == compiled_schema_result -@pytest.mark.parametrize('source, destination', [ - ('./tests/fixtures/books/ymls/', './tests/fixtures/books/results/') -]) +@pytest.mark.parametrize( + "source, destination", + [("./tests/fixtures/books/ymls/", "./tests/fixtures/books/results/")], +) def test_cli_convert_yaml2json(source, destination): - """Test cds-dojson CLI 'convert_yaml2json' command""" + """Test cds-dojson CLI 'convert_yaml2json' command.""" runner = CliRunner() result = runner.invoke(convert_yaml2json, [source, destination]) assert 0 == result.exit_code - with open('./tests/fixtures/books/books_title_mock.json') as s: + with open("./tests/fixtures/books/books_title_mock.json") as s: # read the mock JSON file json_mock = json.load(s) - with open(os.path.join(destination, 'books_title.json')) as s: + with open(os.path.join(destination, "books_title.json")) as s: # read the newly created JSON file json_converted = json.load(s) diff --git a/tests/test_fields_utils.py b/tests/test_fields_utils.py index 4f8a8712..14adea97 100644 --- a/tests/test_fields_utils.py +++ b/tests/test_fields_utils.py @@ -40,41 +40,42 @@ def test_clean_pages(): """Test if pages are extracted properly""" - pages_field = {'a': '12-14'} - assert clean_pages_range('a', pages_field) == {'page_start': 12, - 'page_end': 14} + pages_field = {"a": "12-14"} + assert clean_pages_range("a", pages_field) == {"page_start": 12, "page_end": 14} - pages_field = {'a': '1'} - assert clean_pages_range('a', pages_field) == {'page_start': 1} + pages_field = {"a": "1"} + assert clean_pages_range("a", pages_field) == {"page_start": 1} with pytest.raises(UnexpectedValue): - pages_field = {'a': '12-14-45'} - clean_pages_range('a', pages_field) + pages_field = {"a": "12-14-45"} + clean_pages_range("a", pages_field) def test_rel_url(): """Test if build of related record url works.""" - assert related_url(245432) == 'https://cds.cern.ch/record/245432' + assert related_url(245432) == "https://cds.cern.ch/record/245432" -@pytest.mark.parametrize('to_clean, regex_format, req, output', - [(' BOOK', None, False, 'BOOK'), - (' BOOK ', None, False, 'BOOK'), - ('201845', r'\d{6}$', False, '201845'), - ('20-50', r'\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]\d+)$', - False, '20-50'), - ]) +@pytest.mark.parametrize( + "to_clean, regex_format, req, output", + [ + (" BOOK", None, False, "BOOK"), + (" BOOK ", None, False, "BOOK"), + ("201845", r"\d{6}$", False, "201845"), + ("20-50", r"\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]\d+)$", False, "20-50"), + ], +) def test_clean_str(to_clean, regex_format, req, output): """Test if clean str works correctly""" assert clean_str(to_clean, regex_format, req) == output - assert clean_str('TEST', None, False, transform='lower') == 'test' + assert clean_str("TEST", None, False, transform="lower") == "test" @pytest.mark.xfail(raises=MissingRequiredField) def test_clean_str_required(): """Test if it fails when require val is needed""" - assert clean_str('', None, req=True) + assert clean_str("", None, req=True) @pytest.mark.xfail(raises=UnexpectedValue) @@ -86,63 +87,89 @@ def test_clean_str_none(): @pytest.mark.xfail(raises=UnexpectedValue) def test_clean_str_regex(): """Test if it fails when require val is needed""" - assert clean_str('300500', r'\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]\d+)$', req=True) + assert clean_str("300500", r"\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]\d+)$", req=True) @pytest.mark.parametrize( - 'subfield, value, var_type, req, default, manual, output', - [('a', {'a': 'CERN'}, str, False, None, None, 'CERN'), - ('a', {'a': '12345'}, int, False, None, None, 12345), - ('a', {'a': ''}, bool, False, None, None, False), - ('a', {}, str, True, 'TEST', None, 'TEST'), - ('a', {}, int, True, 100, None, 100), ]) + "subfield, value, var_type, req, default, manual, output", + [ + ("a", {"a": "CERN"}, str, False, None, None, "CERN"), + ("a", {"a": "12345"}, int, False, None, None, 12345), + ("a", {"a": ""}, bool, False, None, None, False), + ("a", {}, str, True, "TEST", None, "TEST"), + ("a", {}, int, True, 100, None, 100), + ], +) def test_clean_val(subfield, value, var_type, req, default, manual, output): """Test if clean value works properly""" - assert clean_val(subfield, value, var_type, req=req, - default=default, manual=manual) == output + assert ( + clean_val(subfield, value, var_type, req=req, default=default, manual=manual) + == output + ) @pytest.mark.parametrize( - 'subfield, value, var_type, req, default, manual, regexp, output', - [('a', {'a': '20-40'}, str, False, None, None, - r'\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]\d+)$', '20-40'), - ]) -def test_clean_val_regexp(subfield, value, var_type, req, default, manual, - regexp, output): - assert clean_val(subfield, value, var_type, req=req, - default=default, manual=manual, - regex_format=regexp) == output + "subfield, value, var_type, req, default, manual, regexp, output", + [ + ( + "a", + {"a": "20-40"}, + str, + False, + None, + None, + r"\d+(?:[\-‐‑‒–—―⁻₋−﹘﹣－]\d+)$", + "20-40", + ), + ], +) +def test_clean_val_regexp( + subfield, value, var_type, req, default, manual, regexp, output +): + assert ( + clean_val( + subfield, + value, + var_type, + req=req, + default=default, + manual=manual, + regex_format=regexp, + ) + == output + ) @pytest.mark.xfail(raises=ManualMigrationRequired) def test_clean_val_manual(): """Test if manual exception is raised properly""" - assert clean_val('a', {'a': 'CERN'}, str, manual=True) + assert clean_val("a", {"a": "CERN"}, str, manual=True) @pytest.mark.xfail(raises=MissingRequiredField) def test_clean_val_required(): """Tests if fails when required value missing""" - assert clean_val('a', {}, str, req=True) + assert clean_val("a", {}, str, req=True) @pytest.mark.xfail(raises=NotImplementedError) def test_clean_val_type(): """Tests if fails when unexpected value type""" - assert clean_val('a', {'a': 'CERN'}, float) == 'CERN' + assert clean_val("a", {"a": "CERN"}, float) == "CERN" def test_clean_email(): """Test if the email is cleaned properly""" - assert clean_email('johndoe@cern.ch') == 'johndoe@cern.ch' - assert clean_email('johndoe[CERN]') == 'johndoe@cern.ch' - assert clean_email('johndoe [CERN]') == 'johndoe@cern.ch' + assert clean_email("johndoe@cern.ch") == "johndoe@cern.ch" + assert clean_email("johndoe[CERN]") == "johndoe@cern.ch" + assert clean_email("johndoe [CERN]") == "johndoe@cern.ch" def test_get_week_start(): """Test if the week start date is calculated properly""" from datetime import date + assert get_week_start(2018, 20) == date(2018, 5, 14) assert get_week_start(2017, 5) == date(2017, 1, 30) assert get_week_start(2016, 52) == date(2016, 12, 26) @@ -151,12 +178,14 @@ def test_get_week_start(): @pytest.mark.parametrize( - 'phrase, replace_with, key, value, output', - [('Collaboration', '', None, ['ATLAS Collaboration'], ['ATLAS']), - ('ed.', 'editor', None, ['main ed.'], ['main editor']), - ('ed.', 'editor', 'role', [{'role': 'ed.'}], [{'role': 'editor'}]), - ('ed.', 'editor', None, [], []), - ]) + "phrase, replace_with, key, value, output", + [ + ("Collaboration", "", None, ["ATLAS Collaboration"], ["ATLAS"]), + ("ed.", "editor", None, ["main ed."], ["main editor"]), + ("ed.", "editor", "role", [{"role": "ed."}], [{"role": "editor"}]), + ("ed.", "editor", None, [], []), + ], +) def test_replace_in_result(phrase, replace_with, key, value, output): """Test if the replace in result decorator works""" @@ -174,9 +203,9 @@ def test_filter_list_values(): def func(self, key, value): return value - assert func(None, 'key', - [{'key': 'test', 'p': '', 'o': None, - 's': [], 'k':{}}]) == [{'key': 'test'}] + assert func( + None, "key", [{"key": "test", "p": "", "o": None, "s": [], "k": {}}] + ) == [{"key": "test"}] @pytest.mark.xfail(raises=IgnoreKey) @@ -187,7 +216,7 @@ def test_filter_list_ignore(): def func(self, key, value): return value - func(None, 'key', [{'key': ''}]) + func(None, "key", [{"key": ""}]) @pytest.mark.xfail(raises=IgnoreKey) @@ -198,16 +227,19 @@ def test_filter_list_ignore_empty(): def func(self, key, value): return value - func(None, 'key', []) + func(None, "key", []) -@pytest.mark.parametrize('value_in, out', - [ - (['test1', 'test2', ''], ['test1', 'test2']), - ('test', 'test'), - ]) +@pytest.mark.parametrize( + "value_in, out", + [ + (["test1", "test2", ""], ["test1", "test2"]), + ("test", "test"), + ], +) def test_out_strip(value_in, out): """Test if out strip works properly""" + @out_strip def func(self, key, value): return value @@ -244,4 +276,4 @@ def test_out_ignore_type(): def func(self, key, value): return value - assert func(None, None, {'p': 'zzz'}) == {'p': 'zzz'} + assert func(None, None, {"p": "zzz"}) == {"p": "zzz"} diff --git a/tests/test_journals.py b/tests/test_journals.py index 9d052b09..d5046467 100644 --- a/tests/test_journals.py +++ b/tests/test_journals.py @@ -23,25 +23,27 @@ from cds_dojson.marc21.models.books.journal import model from cds_dojson.marc21.utils import create_record -marcxml = ("""""" - """{0}""") +marcxml = ( + """""" + """{0}""" +) def check_transformation(marcxml_body, json_body): blob = create_record(marcxml.format(marcxml_body)) record = model.do(blob, ignore_missing=False) expected = { - '$schema': 'https://127.0.0.1:5000/schemas/series/series-v1.0.0.json', - '_migration': { - 'is_multipart': False, - 'has_related': False, - 'record_type': 'journal', - 'volumes': [], - 'items': [], - 'electronic_items': [], - 'relation_previous': None, - 'relation_next': None, - } + "$schema": "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json", + "_migration": { + "is_multipart": False, + "has_related": False, + "record_type": "journal", + "volumes": [], + "items": [], + "electronic_items": [], + "relation_previous": None, + "relation_next": None, + }, } expected.update(**json_body) assert record == expected @@ -57,13 +59,14 @@ def test_title(app): """, { - 'title': 'Radioengineering', - 'alternative_titles': [ + "title": "Radioengineering", + "alternative_titles": [ { - 'value': "Proceedings of Czech and Slovak Technical Universities", - 'type': 'SUBTITLE'}, - ] - } + "value": "Proceedings of Czech and Slovak Technical Universities", + "type": "SUBTITLE", + }, + ], + }, ) @@ -79,13 +82,14 @@ def test_alternative_titles(app): """, { - 'title': 'Recherche Coopérative sur Programme no 25', - 'alternative_titles': [ + "title": "Recherche Coopérative sur Programme no 25", + "alternative_titles": [ { - 'value': "Les rencontres physiciens-mathématiciens de Strasbourg", - 'type': 'ALTERNATIVE_TITLE'}, - ] - } + "value": "Les rencontres physiciens-mathématiciens de Strasbourg", + "type": "ALTERNATIVE_TITLE", + }, + ], + }, ) @@ -98,8 +102,8 @@ def test_abbreviated_title(app): """, { - 'abbreviated_title': 'Bull. CERN Comm.', - } + "abbreviated_title": "Bull. CERN Comm.", + }, ) @@ -113,9 +117,10 @@ def test_identifiers(app): """, { - 'identifiers': [{'scheme': 'ISSN', 'value': '1805-9600', - 'material': 'online'}], - } + "identifiers": [ + {"scheme": "ISSN", "value": "1805-9600", "material": "online"} + ], + }, ) @@ -128,9 +133,8 @@ def test_internal_notes(app): """, { - 'internal_notes': [ - {'value': 'Online archives purchased 2014'}], - } + "internal_notes": [{"value": "Online archives purchased 2014"}], + }, ) @@ -143,8 +147,8 @@ def test_note(app): """, { - 'note': 'The reports are catalogued individually and retrievable in the catalogue.', - } + "note": "The reports are catalogued individually and retrievable in the catalogue.", + }, ) @@ -158,8 +162,8 @@ def test_publisher(app): """, { - 'publisher': 'National Council on Radiation Protection and Measurements', - } + "publisher": "National Council on Radiation Protection and Measurements", + }, ) @@ -172,8 +176,8 @@ def test_languages(app): """, { - 'languages': ['EN'], - } + "languages": ["EN"], + }, ) @@ -186,21 +190,17 @@ def test_migration(app): """, { - '_migration': { - 'is_multipart': False, - 'has_related': False, - 'record_type': 'journal', - 'volumes': [], - 'electronic_items': [ - { - 'subscription': 'v 1 (1992) -' - } - ], - 'items': [], - 'relation_next': None, - 'relation_previous': None, + "_migration": { + "is_multipart": False, + "has_related": False, + "record_type": "journal", + "volumes": [], + "electronic_items": [{"subscription": "v 1 (1992) -"}], + "items": [], + "relation_next": None, + "relation_previous": None, }, - } + }, ) check_transformation( """ @@ -212,24 +212,24 @@ def test_migration(app): """, { - '_migration': { - 'is_multipart': False, - 'has_related': False, - 'record_type': 'journal', - 'volumes': [], - 'electronic_items': [ + "_migration": { + "is_multipart": False, + "has_related": False, + "record_type": "journal", + "volumes": [], + "electronic_items": [ { - 'subscription': 'v 1 (1992) -', - 'url': 'https://www.radioeng.cz/search.htm', - 'access_type': '6', - 'note': 'tada', + "subscription": "v 1 (1992) -", + "url": "https://www.radioeng.cz/search.htm", + "access_type": "6", + "note": "tada", } ], - 'items': [], - 'relation_next': None, - 'relation_previous': None, + "items": [], + "relation_next": None, + "relation_previous": None, }, - } + }, ) check_transformation( """ @@ -239,21 +239,21 @@ def test_migration(app): """, { - '_migration': { - 'is_multipart': False, - 'has_related': False, - 'record_type': 'journal', - 'volumes': [], - 'items': [ + "_migration": { + "is_multipart": False, + "has_related": False, + "record_type": "journal", + "volumes": [], + "items": [ { - 'subscription': 'v 1 (1992) -', + "subscription": "v 1 (1992) -", } ], - 'electronic_items': [], - 'relation_next': None, - 'relation_previous': None, + "electronic_items": [], + "relation_next": None, + "relation_previous": None, }, - } + }, ) check_transformation( """ @@ -264,17 +264,17 @@ def test_migration(app): """, { - '_migration': { - 'is_multipart': False, - 'has_related': False, - 'record_type': 'journal', - 'volumes': [], - 'items': [], - 'electronic_items': [], - 'relation_next': None, - 'relation_previous': '2711756', + "_migration": { + "is_multipart": False, + "has_related": False, + "record_type": "journal", + "volumes": [], + "items": [], + "electronic_items": [], + "relation_next": None, + "relation_previous": "2711756", }, - } + }, ) check_transformation( """ @@ -285,15 +285,15 @@ def test_migration(app): """, { - '_migration': { - 'is_multipart': False, - 'has_related': False, - 'record_type': 'journal', - 'volumes': [], - 'items': [], - 'electronic_items': [], - 'relation_next': '2711769', - 'relation_previous': None, + "_migration": { + "is_multipart": False, + "has_related": False, + "record_type": "journal", + "volumes": [], + "items": [], + "electronic_items": [], + "relation_next": "2711769", + "relation_previous": None, }, - } + }, ) diff --git a/tests/test_matcher.py b/tests/test_matcher.py index 3e5abc9f..e51f687f 100644 --- a/tests/test_matcher.py +++ b/tests/test_matcher.py @@ -16,80 +16,82 @@ # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA. +"""Matcher tests.""" + import pytest -from dojson.contrib import marc21 as default +from cds_dojson.exceptions import ModelMissingException from cds_dojson.marc21.models.books import book, journal, multipart, serial, standard from cds_dojson.marc21.models.videos import project, video from cds_dojson.matcher import matcher def test_marc21_matcher_videos(): - """Test CDS DoJSON matcher - videos""" - video_blob1 = {'980__': [{'a': 'PUBLVIDEOMOVIE'}, {'b': 'VIDEOMEDIALAB'}]} - video_blob2 = {'980__': [{'a': 'PUBLVIDEOMOVIE'}, {'c': 'DELETED'}]} - video_blob3 = {'980__': {'a': 'VIDEOARC'}} - video_blob4 = {'980__': {'a': 'CERNVIDEOSHOOT'}} - video_blob5 = {'980__': [{'a': 'PUBLVIDEOMOVIE'}, {'b': 'VIDEOMEDIALAB'}], - '970__': {'a': 'AVW.project.1234'}} - video_blob6 = {'980__': [{'a': 'PUBLVIDEOMOVIE'}, {'c': 'DELETED'}], - '970__': {'a': 'AVW.project.1234'}} - video_blob7 = {'970__': {'a': 'FCS.project.987'}} - not_match = {'foo': 'bar'} + """Test CDS DoJSON matcher - videos.""" + video_blob1 = {"980__": [{"a": "PUBLVIDEOMOVIE"}, {"b": "VIDEOMEDIALAB"}]} + video_blob2 = {"980__": [{"a": "PUBLVIDEOMOVIE"}, {"c": "DELETED"}]} + video_blob3 = {"980__": {"a": "VIDEOARC"}} + video_blob4 = {"980__": {"a": "CERNVIDEOSHOOT"}} + video_blob5 = { + "980__": [{"a": "PUBLVIDEOMOVIE"}, {"b": "VIDEOMEDIALAB"}], + "970__": {"a": "AVW.project.1234"}, + } + video_blob6 = { + "980__": [{"a": "PUBLVIDEOMOVIE"}, {"c": "DELETED"}], + "970__": {"a": "AVW.project.1234"}, + } + video_blob7 = {"970__": {"a": "FCS.project.987"}} + not_match = {"foo": "bar"} + + assert video.model == matcher(video_blob1, "cds_dojson.marc21.models") + assert project.model == matcher(video_blob5, "cds_dojson.marc21.models") + assert project.model == matcher(video_blob7, "cds_dojson.marc21.models") - assert video.model == matcher(video_blob1, 'cds_dojson.marc21.models') - assert default.model == matcher(video_blob2, 'cds_dojson.marc21.models') - assert default.model == matcher(video_blob3, 'cds_dojson.marc21.models') - assert default.model == matcher(video_blob4, 'cds_dojson.marc21.models') - assert project.model == matcher(video_blob5, 'cds_dojson.marc21.models') - assert default.model == matcher(video_blob6, 'cds_dojson.marc21.models') - assert project.model == matcher(video_blob7, 'cds_dojson.marc21.models') - assert default.model == matcher(not_match, 'cds_dojson.marc21.models') + for blob in (video_blob2, video_blob3, video_blob4, video_blob6, not_match): + with pytest.raises(ModelMissingException): + matcher(blob, "cds_dojson.marc21.models") def test_marc21_matcher_books(): """Test CDS DoJSON matcher - books.""" - book_blob1 = {'690C_': [{'a': 'BOOK'}]} - book_blob2 = {'980__': [{'a': 'PROCEEDINGS'}]} - book_blob3 = {'697C_': [{'a': 'ENGLISH BOOK CLUB'}]} - serial_blob1 = {'690C_': [{'a': 'BOOK'}], - '490__': {'a': 'Test title'}} - serial_blob2 = {'690C_': [{'a': 'BOOK'}]} + book_blob1 = {"690C_": [{"a": "BOOK"}]} + book_blob2 = {"980__": [{"a": "PROCEEDINGS"}]} + book_blob3 = {"697C_": [{"a": "ENGLISH BOOK CLUB"}]} + serial_blob1 = {"690C_": [{"a": "BOOK"}], "490__": {"a": "Test title"}} + serial_blob2 = {"690C_": [{"a": "BOOK"}]} multipart_blob1 = { - '690C_': [{'a': 'BOOK'}], - '245__': [{'a': 'Test '}], - '596__': [{'a': 'MULTIVOLUMES'}], - '246__': [{'p': 'Volume Title', 'n': '2'}] + "690C_": [{"a": "BOOK"}], + "245__": [{"a": "Test "}], + "596__": [{"a": "MULTIVOLUMES"}], + "246__": [{"p": "Volume Title", "n": "2"}], } multipart_blob2 = { - '690C_': [{'a': 'BOOK'}], - '245__': [{'a': 'Test '}], - '246__': [{'n': '2'}], - '596__': [{'a': 'MULTIVOLUMES'}], + "690C_": [{"a": "BOOK"}], + "245__": [{"a": "Test "}], + "246__": [{"n": "2"}], + "596__": [{"a": "MULTIVOLUMES"}], } - standard_blob1 = {'690C_': [{'a': 'STANDARD'}]} - journal_blob = {'980__': [{'a': 'PERI'}]} - not_match = {'foo': 'bar'} + standard_blob1 = {"690C_": [{"a": "STANDARD"}]} + journal_blob = {"980__": [{"a": "PERI"}]} + not_match = {"foo": "bar"} - assert book.model == matcher(book_blob1, 'cds_dojson.marc21.models') - assert book.model == matcher(book_blob2, 'cds_dojson.marc21.models') + assert book.model == matcher(book_blob1, "cds_dojson.marc21.models") + assert book.model == matcher(book_blob2, "cds_dojson.marc21.models") + assert standard.model == matcher(standard_blob1, "cds_dojson.marc21.models") + assert serial.model == matcher(serial_blob1, "cds_dojson.marc21.parent_models") + assert multipart.model == matcher( + multipart_blob1, "cds_dojson.marc21.parent_models" + ) + assert multipart.model == matcher( + multipart_blob2, "cds_dojson.marc21.parent_models" + ) + assert journal.model == matcher(journal_blob, "cds_dojson.marc21.parent_models") - with pytest.raises(AssertionError): - # English book club should not be matched - assert book.model == matcher(book_blob3, 'cds_dojson.marc21.models') - assert standard.model == matcher(standard_blob1, - 'cds_dojson.marc21.models') - assert serial.model == matcher(serial_blob1, - 'cds_dojson.marc21.parent_models' - ) - assert multipart.model == matcher(multipart_blob1, - 'cds_dojson.marc21.parent_models') - assert multipart.model == matcher(multipart_blob2, - 'cds_dojson.marc21.parent_models') - assert journal.model == matcher(journal_blob, - 'cds_dojson.marc21.parent_models') + # English book club should not be matched + with pytest.raises(ModelMissingException): + matcher(book_blob3, "cds_dojson.marc21.models") # make sure that it won't match if 490 not there - assert default.model == matcher(serial_blob2, - 'cds_dojson.marc21.parent_models' - ) - assert default.model == matcher(not_match, 'cds_dojson.marc21.models') + with pytest.raises(ModelMissingException): + matcher(serial_blob2, "cds_dojson.marc21.parent_models") + with pytest.raises(ModelMissingException): + matcher(not_match, "cds_dojson.marc21.models") diff --git a/tests/test_series.py b/tests/test_series.py index 44739151..89c88311 100644 --- a/tests/test_series.py +++ b/tests/test_series.py @@ -29,8 +29,10 @@ from cds_dojson.marc21.models.books.serial import model as serial_model from cds_dojson.marc21.utils import create_record -marcxml = ("""""" - """{0}""") +marcxml = ( + """""" + """{0}""" +) def check_transformation(marcxml_body, json_body, model=None): @@ -38,7 +40,7 @@ def check_transformation(marcxml_body, json_body, model=None): blob = create_record(marcxml.format(marcxml_body)) record = model.do(blob, ignore_missing=False) expected = { - '$schema': 'https://127.0.0.1:5000/schemas/series/series-v1.0.0.json', + "$schema": "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json", } expected.update(**json_body) assert record == expected @@ -55,14 +57,11 @@ def test_serial(app): """, { - 'title': - [ - 'Esl and applied linguistics professional' - ], - 'mode_of_issuance': 'SERIAL', - '_migration': {'record_type': 'serial', 'children': []} + "title": ["Esl and applied linguistics professional"], + "mode_of_issuance": "SERIAL", + "_migration": {"record_type": "serial", "children": []}, }, - serial_model + serial_model, ) check_transformation( @@ -73,16 +72,12 @@ def test_serial(app): """, { - 'title': - [ - 'Springerbriefs in history' - ' of science and technology' - ], - 'mode_of_issuance': 'SERIAL', - 'identifiers': [{'scheme': 'ISSN', 'value': '2211-4564'}], - '_migration': {'record_type': 'serial', 'children': []} + "title": ["Springerbriefs in history" " of science and technology"], + "mode_of_issuance": "SERIAL", + "identifiers": [{"scheme": "ISSN", "value": "2211-4564"}], + "_migration": {"record_type": "serial", "children": []}, }, - serial_model + serial_model, ) check_transformation( @@ -93,16 +88,12 @@ def test_serial(app): """, { - 'title': - [ - 'Springerbriefs in history' - ' of science and technology' - ], - 'mode_of_issuance': 'SERIAL', - 'identifiers': [{'scheme': 'ISSN', 'value': '2211-4564'}], - '_migration': {'record_type': 'serial', 'children': []} + "title": ["Springerbriefs in history" " of science and technology"], + "mode_of_issuance": "SERIAL", + "identifiers": [{"scheme": "ISSN", "value": "2211-4564"}], + "_migration": {"record_type": "serial", "children": []}, }, - serial_model + serial_model, ) with pytest.raises(MissingRule): @@ -113,14 +104,10 @@ def test_serial(app): """, { - 'title': - [ - 'Springerbriefs in history' - ' of science and technology' - ], - '_migration': {'record_type': 'serial', 'children': []} + "title": ["Springerbriefs in history" " of science and technology"], + "_migration": {"record_type": "serial", "children": []}, }, - serial_model + serial_model, ) @@ -147,26 +134,22 @@ def test_monograph(app): """, { - 'title': 'La fisica di Amaldi', - 'alternative_titles': [ - { - 'value': 'idee ed esperimenti : con CD-ROM', - 'type': 'SUBTITLE' - } + "title": "La fisica di Amaldi", + "alternative_titles": [ + {"value": "idee ed esperimenti : con CD-ROM", "type": "SUBTITLE"} ], - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - 'number_of_volumes': '2', - '_migration': { + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "number_of_volumes": "2", + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', 'volumes': [ - {'title': 'Introduzione alla fisica meccanica', - 'volume': '1'}, - {'title': 'Termologia, onde, relatività', - 'volume': '2'} - ]} - + "record_type": "multipart", + "volumes": [ + {"title": "Introduzione alla fisica meccanica", "volume": "1"}, + {"title": "Termologia, onde, relatività", "volume": "2"}, + ], + }, }, - multipart_model + multipart_model, ) check_transformation( @@ -184,22 +167,19 @@ def test_monograph(app): """, { - 'title': 'La fisica di Amaldi', - 'alternative_titles': [ - { - 'value': 'idee ed esperimenti : con CD-ROM', - 'type': 'SUBTITLE' - } + "title": "La fisica di Amaldi", + "alternative_titles": [ + {"value": "idee ed esperimenti : con CD-ROM", "type": "SUBTITLE"} ], - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - 'number_of_volumes': '2', - '_migration': { + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "number_of_volumes": "2", + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', 'volumes': [ - {'title': 'Termologia, onde, relatività', - 'volume': '2'} - ]} - + "record_type": "multipart", + "volumes": [ + {"title": "Termologia, onde, relatività", "volume": "2"} + ], + }, }, multipart_model, ) @@ -219,20 +199,18 @@ def test_monograph(app): """, { - 'title': 'La fisica di Amaldi', - 'alternative_titles': [ - { - 'value': 'idee ed esperimenti : con CD-ROM', - 'type': 'SUBTITLE' - } + "title": "La fisica di Amaldi", + "alternative_titles": [ + {"value": "idee ed esperimenti : con CD-ROM", "type": "SUBTITLE"} ], - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - '_migration': { + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', 'volumes': [ - {'title': 'Termologia, onde, relatività', - 'volume': '2'} - ]} + "record_type": "multipart", + "volumes": [ + {"title": "Termologia, onde, relatività", "volume": "2"} + ], + }, }, multipart_model, ) @@ -253,19 +231,18 @@ def test_monograph(app): """, { - 'title': 'La fisica di Amaldi', - 'alternative_titles': [ - { - 'value': 'idee ed esperimenti : con CD-ROM', - 'type': 'SUBTITLE' - } + "title": "La fisica di Amaldi", + "alternative_titles": [ + {"value": "idee ed esperimenti : con CD-ROM", "type": "SUBTITLE"} ], - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - '_migration': { + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', 'volumes': [ - {'title': 'Termologia, onde, relatività', 'volume': '2'} - ]} + "record_type": "multipart", + "volumes": [ + {"title": "Termologia, onde, relatività", "volume": "2"} + ], + }, }, multipart_model, ) @@ -286,16 +263,20 @@ def test_monograph(app): """, { - 'title': 'La fisica di Amaldi', - 'alternative_titles': [ - {'value': 'idee ed esperimenti : con CD-ROM', - 'type': 'SUBTITLE'} + "title": "La fisica di Amaldi", + "alternative_titles": [ + { + "value": "idee ed esperimenti : con CD-ROM", + "type": "SUBTITLE", + } ], - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - 'number_of_volumes': '2', - '_migration': { + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "number_of_volumes": "2", + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', 'volumes': []} + "record_type": "multipart", + "volumes": [], + }, }, multipart_model, ) @@ -312,14 +293,20 @@ def test_monograph(app): """, { - 'title': 'La fisica di Amaldi', - 'alternative_titles': [ - {'value': 'idee ed esperimenti : con CD-ROM', - 'type': 'SUBTITLE'} + "title": "La fisica di Amaldi", + "alternative_titles": [ + { + "value": "idee ed esperimenti : con CD-ROM", + "type": "SUBTITLE", + } ], - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - 'number_of_volumes': '2', - '_migration': {**get_migration_dict(), 'record_type': 'multipart', 'volumes': []} + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "number_of_volumes": "2", + "_migration": { + **get_migration_dict(), + "record_type": "multipart", + "volumes": [], + }, }, multipart_model, ) @@ -377,71 +364,59 @@ def test_monograph_migration(app): """, { - '_migration': { + "_migration": { **get_migration_dict(), - 'volumes': [ + "volumes": [ { - 'volume': '3', - 'isbn': '1108052819', - 'physical_description': 'print version, paperback', - 'is_electronic': False, + "volume": "3", + "isbn": "1108052819", + "physical_description": "print version, paperback", + "is_electronic": False, }, { - 'volume': '3', - 'isbn': '9781108052818', - 'physical_description': - 'print version, paperback', - 'is_electronic': False, + "volume": "3", + "isbn": "9781108052818", + "physical_description": "print version, paperback", + "is_electronic": False, }, { - 'volume': '2', - 'isbn': '9781108052801', - 'physical_description': - 'print version, paperback', - 'is_electronic': False, + "volume": "2", + "isbn": "9781108052801", + "physical_description": "print version, paperback", + "is_electronic": False, }, { - 'volume': '2', - 'isbn': '1108052800', - 'physical_description': - 'print version, paperback', - 'is_electronic': False, + "volume": "2", + "isbn": "1108052800", + "physical_description": "print version, paperback", + "is_electronic": False, }, { - 'volume': '1', - 'isbn': '9781108052795', - 'physical_description': - 'print version, paperback', - 'is_electronic': False, + "volume": "1", + "isbn": "9781108052795", + "physical_description": "print version, paperback", + "is_electronic": False, }, { - 'volume': '1', - 'isbn': '1108052797', - 'physical_description': - 'print version, paperback', - 'is_electronic': False, - }, - { - 'title': '1865-1874', - 'volume': '1' - }, - { - 'title': '1875-1881', - 'volume': '2' - }, - { - 'title': '1882-1905', - 'volume': '3' + "volume": "1", + "isbn": "1108052797", + "physical_description": "print version, paperback", + "is_electronic": False, }, + {"title": "1865-1874", "volume": "1"}, + {"title": "1875-1881", "volume": "2"}, + {"title": "1882-1905", "volume": "3"}, ], - 'record_type': 'multipart', + "record_type": "multipart", }, - 'title': 'Wissenschaftliche Abhandlungen', - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - 'number_of_volumes': '3', - 'identifiers': [{'scheme': 'ISBN', 'value': '9781108052825'}], - 'physical_description': 'print version, paperback', - }, multipart_model) + "title": "Wissenschaftliche Abhandlungen", + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "number_of_volumes": "3", + "identifiers": [{"scheme": "ISBN", "value": "9781108052825"}], + "physical_description": "print version, paperback", + }, + multipart_model, + ) def test_monograph_invalid_volume_migration(app): @@ -462,7 +437,9 @@ def test_monograph_invalid_volume_migration(app): print version, paperback (v.2, CD-ROM) """, - {}, multipart_model) + {}, + multipart_model, + ) def test_monograph_volume_migration_no_description(app): @@ -475,13 +452,18 @@ def test_monograph_volume_migration_no_description(app): v.13 """, - {'_migration': { - **get_migration_dict(), - 'record_type': 'multipart', - 'volumes': [{'is_electronic': False, - 'isbn': '1560810726', - 'volume': '13'}]}, - 'mode_of_issuance': 'MULTIPART_MONOGRAPH'}, multipart_model) + { + "_migration": { + **get_migration_dict(), + "record_type": "multipart", + "volumes": [ + {"is_electronic": False, "isbn": "1560810726", "volume": "13"} + ], + }, + "mode_of_issuance": "MULTIPART_MONOGRAPH", + }, + multipart_model, + ) def test_monograph_with_electronic_isbns(app): @@ -521,45 +503,45 @@ def test_monograph_with_electronic_isbns(app): """, { - 'identifiers': [ - {'scheme': 'ISBN', 'value': '0817631852'}, - {'scheme': 'ISBN', 'value': '0817631879'}, - {'scheme': 'ISBN', 'value': '9780817631857'}, - {'scheme': 'ISBN', 'value': '9780817631871'}, - + "identifiers": [ + {"scheme": "ISBN", "value": "0817631852"}, + {"scheme": "ISBN", "value": "0817631879"}, + {"scheme": "ISBN", "value": "9780817631857"}, + {"scheme": "ISBN", "value": "9780817631871"}, ], - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - '_migration': { + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', - 'volumes': [ + "record_type": "multipart", + "volumes": [ { - 'is_electronic': False, - 'physical_description': 'print version', - 'volume': '2', - 'isbn': '0817631852' + "is_electronic": False, + "physical_description": "print version", + "volume": "2", + "isbn": "0817631852", }, { - 'is_electronic': True, - 'physical_description': 'electronic version', - 'volume': '2', - 'isbn': '9781461239406' + "is_electronic": True, + "physical_description": "electronic version", + "volume": "2", + "isbn": "9781461239406", }, { - 'is_electronic': True, - 'physical_description': 'electronic version', - 'volume': '1', - 'isbn': '9781461251545' + "is_electronic": True, + "physical_description": "electronic version", + "volume": "1", + "isbn": "9781461251545", }, { - 'is_electronic': False, - 'physical_description': 'print version', - 'volume': '1', - 'isbn': '9781461295891' + "is_electronic": False, + "physical_description": "print version", + "volume": "1", + "isbn": "9781461295891", }, ], }, - }, multipart_model + }, + multipart_model, ) @@ -575,20 +557,22 @@ def test_monograph_volume_migration_doi(app): """, { - '_migration': - { + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', - 'volumes': [{ - 'doi': '10.1007/978-3-030-49613-5', - 'material': 'ebook', - 'source': None, - 'volume': '1' - }] + "record_type": "multipart", + "volumes": [ + { + "doi": "10.1007/978-3-030-49613-5", + "material": "ebook", + "source": None, + "volume": "1", + } + ], }, - 'mode_of_issuance': 'MULTIPART_MONOGRAPH' + "mode_of_issuance": "MULTIPART_MONOGRAPH", }, - multipart_model) + multipart_model, + ) def test_monograph_volume_barcode(app): @@ -602,15 +586,15 @@ def test_monograph_volume_barcode(app): """, { - '_migration': - { + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', - 'volumes': [{'barcode': '73-0089-0', 'volume': 'A'}] + "record_type": "multipart", + "volumes": [{"barcode": "73-0089-0", "volume": "A"}], }, - 'mode_of_issuance': 'MULTIPART_MONOGRAPH' + "mode_of_issuance": "MULTIPART_MONOGRAPH", }, - multipart_model) + multipart_model, + ) def test_monograph_volume_url(app): @@ -624,18 +608,21 @@ def test_monograph_volume_url(app): """, { - '_migration': - { + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', - 'volumes': [{ - 'description': 'ebook', - 'url': 'https://cds.cern.ch/auth.py?r=EBLIB_P_1890382', - 'volume': '1'}] + "record_type": "multipart", + "volumes": [ + { + "description": "ebook", + "url": "https://cds.cern.ch/auth.py?r=EBLIB_P_1890382", + "volume": "1", + } + ], }, - 'mode_of_issuance': 'MULTIPART_MONOGRAPH' + "mode_of_issuance": "MULTIPART_MONOGRAPH", }, - multipart_model) + multipart_model, + ) def test_monograph_legacy_representation(app): @@ -651,16 +638,16 @@ def test_monograph_legacy_representation(app): """, { - '_migration': - { + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', - 'multipart_id': 'Vol965', - 'multivolume_record_format': True, + "record_type": "multipart", + "multipart_id": "Vol965", + "multivolume_record_format": True, }, - 'mode_of_issuance': 'MULTIPART_MONOGRAPH' + "mode_of_issuance": "MULTIPART_MONOGRAPH", }, - multipart_model) + multipart_model, + ) def test_monograph_legacy_report_number(app): @@ -673,15 +660,17 @@ def test_monograph_legacy_report_number(app): """, { - '_migration': - { + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', + "record_type": "multipart", }, - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - 'identifiers': [{'scheme': 'report_number', 'value': 'IAEA-INIS-20-REV-0-D'}] + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "identifiers": [ + {"scheme": "report_number", "value": "IAEA-INIS-20-REV-0-D"} + ], }, - multipart_model) + multipart_model, + ) def test_monograph_series_authors(app): @@ -697,21 +686,16 @@ def test_monograph_series_authors(app): """, { - '_migration': - { + "_migration": { **get_migration_dict(), - 'record_type': 'multipart', - 'authors': [ - { - 'full_name': 'Mehra, Jagdish', - 'roles': ['AUTHOR'] - }, - { - 'full_name': 'Rechenberg, Helmut', - 'roles': ['AUTHOR'] - }], + "record_type": "multipart", + "authors": [ + {"full_name": "Mehra, Jagdish", "roles": ["AUTHOR"]}, + {"full_name": "Rechenberg, Helmut", "roles": ["AUTHOR"]}, + ], }, - 'mode_of_issuance': 'MULTIPART_MONOGRAPH', - 'authors': ['Mehra, Jagdish', 'Rechenberg, Helmut'] + "mode_of_issuance": "MULTIPART_MONOGRAPH", + "authors": ["Mehra, Jagdish", "Rechenberg, Helmut"], }, - multipart_model) + multipart_model, + ) diff --git a/tests/test_standard.py b/tests/test_standard.py index 823e8fb9..2de5d2ae 100644 --- a/tests/test_standard.py +++ b/tests/test_standard.py @@ -26,18 +26,20 @@ from cds_dojson.marc21.models.books.standard import model from cds_dojson.marc21.utils import create_record -marcxml = ("""""" - """{0}""") +marcxml = ( + """""" + """{0}""" +) def check_transformation(marcxml_body, json_body): blob = create_record(marcxml.format(marcxml_body)) - record = {'_migration': {**get_migration_dict()}} + record = {"_migration": {**get_migration_dict()}} record.update(**model.do(blob, ignore_missing=False)) expected = { - '$schema': 'https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json', - '_migration': {**get_migration_dict()} + "$schema": "https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json", + "_migration": {**get_migration_dict()}, } expected.update(**json_body) assert record == expected @@ -53,12 +55,11 @@ def test_title(app): """, { - - 'title': 'Incoterms 2010', - 'alternative_titles': [ - {'value': "les règles de l'ICC", 'type': 'SUBTITLE'}, - ] - } + "title": "Incoterms 2010", + "alternative_titles": [ + {"value": "les règles de l'ICC", "type": "SUBTITLE"}, + ], + }, ) check_transformation( """ @@ -77,21 +78,22 @@ def test_title(app): """, { - 'document_type': 'STANDARD', - 'title': 'Test', - 'alternative_titles': [ - {'value': 'Subtitle', 'type': 'SUBTITLE'}, - {'value': 'Water quality — sampling', - 'type': 'TRANSLATED_TITLE', - 'language': 'fr', - }, + "document_type": "STANDARD", + "title": "Test", + "alternative_titles": [ + {"value": "Subtitle", "type": "SUBTITLE"}, + { + "value": "Water quality — sampling", + "type": "TRANSLATED_TITLE", + "language": "fr", + }, { - 'type': 'TRANSLATED_SUBTITLE', - 'value': u"""part 15: guidance on the preservation and handling of sludge""", - 'language': 'fr', - } + "type": "TRANSLATED_SUBTITLE", + "value": """part 15: guidance on the preservation and handling of sludge""", + "language": "fr", + }, ], - } + }, ) check_transformation( """ @@ -111,23 +113,26 @@ def test_title(app): 373 p - """, { - 'document_type': 'STANDARD', - 'title': 'Test', - 'alternative_titles': [ - {'value': 'Subtitle', 'type': 'SUBTITLE'}, - {'value': 'Water quality — sampling', - 'type': 'TRANSLATED_TITLE', - 'language': 'fr', - }, + """, + { + "document_type": "STANDARD", + "title": "Test", + "alternative_titles": [ + {"value": "Subtitle", "type": "SUBTITLE"}, + { + "value": "Water quality — sampling", + "type": "TRANSLATED_TITLE", + "language": "fr", + }, { - 'type': 'TRANSLATED_SUBTITLE', - 'value': u"""part 15: guidance on the preservation and handling of sludge""", - 'language': 'fr', - } + "type": "TRANSLATED_SUBTITLE", + "value": """part 15: guidance on the preservation and handling of sludge""", + "language": "fr", + }, ], - 'number_of_pages': 373, - }) + "number_of_pages": 373, + }, + ) check_transformation( """ @@ -146,20 +151,22 @@ def test_title(app): mult. p - """, { - 'document_type': 'STANDARD', - 'title': 'Test', - - 'alternative_titles': [ - {'value': 'Subtitle', 'type': 'SUBTITLE'}, - {'value': 'Water quality — sampling', - 'type': 'TRANSLATED_TITLE', - 'language': 'fr', - }, + """, + { + "document_type": "STANDARD", + "title": "Test", + "alternative_titles": [ + {"value": "Subtitle", "type": "SUBTITLE"}, + { + "value": "Water quality — sampling", + "type": "TRANSLATED_TITLE", + "language": "fr", + }, { - 'type': 'TRANSLATED_SUBTITLE', - 'value': u"""part 15: guidance on the preservation and handling of sludge""", - 'language': 'fr', - } + "type": "TRANSLATED_SUBTITLE", + "value": """part 15: guidance on the preservation and handling of sludge""", + "language": "fr", + }, ], - }) + }, + ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 22e49ddb..97d417ba 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -45,26 +45,48 @@ def test_for_each_squash(): @for_each_squash @filter_values def field(self, key, value): - return {'a': value.get('1'), 'b': value.get('2')} + return {"a": value.get("1"), "b": value.get("2")} - squashed = field(None, None, {'1': 'foo', '2': 'bar'}) - assert squashed == {'a': 'foo', 'b': 'bar'} + squashed = field(None, None, {"1": "foo", "2": "bar"}) + assert squashed == {"a": "foo", "b": "bar"} - squashed = field(None, None, [{'1': 'foo'}, {'2': 'bar'}]) - assert squashed == {'a': 'foo', 'b': 'bar'} + squashed = field(None, None, [{"1": "foo"}, {"2": "bar"}]) + assert squashed == {"a": "foo", "b": "bar"} - squashed = field(None, None, [{'1': 'foo', '2': 'bar2'}, {'2': 'bar'}]) - assert squashed == {'a': 'foo', 'b': ['bar2', 'bar']} + squashed = field(None, None, [{"1": "foo", "2": "bar2"}, {"2": "bar"}]) + assert squashed == {"a": "foo", "b": ["bar2", "bar"]} def test_convert_date_to_iso_8601(): """Check if convert_date_to_iso_8601 works correctly""" - string_dates = ('14/12/1989', '2013-11-22', '1999', 'Sep 1970', - '08 May 2002', '25 Feb 2009', '13 Dec 2002', - '23 August 2005', '1989/12/14', '10/09/13', '', None, ) - iso_dates = ('1989-12-14', '2013-11-22', '1999-01-01', '1970-09-01', - '2002-05-08', '2009-02-25', '2002-12-13', '2005-08-23', - '1989-12-14', '2013-09-10', '', None, ) + string_dates = ( + "14/12/1989", + "2013-11-22", + "1999", + "Sep 1970", + "08 May 2002", + "25 Feb 2009", + "13 Dec 2002", + "23 August 2005", + "1989/12/14", + "10/09/13", + "", + None, + ) + iso_dates = ( + "1989-12-14", + "2013-11-22", + "1999-01-01", + "1970-09-01", + "2002-05-08", + "2009-02-25", + "2002-12-13", + "2005-08-23", + "1989-12-14", + "2013-09-10", + "", + None, + ) for string_date, iso_date in zip(string_dates, iso_dates): assert convert_date_to_iso_8601(string_date) == iso_date @@ -76,87 +98,84 @@ def test_not_accessed_keys(): d1 = MementoDict([]) assert not not_accessed_keys(d1) - d1 = MementoDict([('a', [1, 2, 3])]) - assert not_accessed_keys(d1) == {'a'} - - d1 = MementoDict([ - ('a', [1, 2, 3]), - ('b', MementoDict({'1': 1})) - ]) - assert not_accessed_keys(d1) == {'a', 'b1'} - - d1 = MementoDict([ - ('a', [1, 2, 3]), - ('b', MementoDict({'1': 1})), - ('c', 1) - ]) - assert not_accessed_keys(d1) == {'a', 'b1', 'c'} - - d1 = MementoDict([ - ('a', [1, 2, 3]), - ('b', MementoDict({'1': 1})), - ('c', 1), - ('d', [MementoDict({'1': 1, '2': 2}), MementoDict({'1': 2, '2': 2})]) - ]) - assert not_accessed_keys(d1) == {'a', 'b1', 'c', 'd1', 'd2'} + d1 = MementoDict([("a", [1, 2, 3])]) + assert not_accessed_keys(d1) == {"a"} + + d1 = MementoDict([("a", [1, 2, 3]), ("b", MementoDict({"1": 1}))]) + assert not_accessed_keys(d1) == {"a", "b1"} + + d1 = MementoDict([("a", [1, 2, 3]), ("b", MementoDict({"1": 1})), ("c", 1)]) + assert not_accessed_keys(d1) == {"a", "b1", "c"} + + d1 = MementoDict( + [ + ("a", [1, 2, 3]), + ("b", MementoDict({"1": 1})), + ("c", 1), + ("d", [MementoDict({"1": 1, "2": 2}), MementoDict({"1": 2, "2": 2})]), + ] + ) + assert not_accessed_keys(d1) == {"a", "b1", "c", "d1", "d2"} assert not d1.accessed_keys - d1['d'][0]['1'] - assert not_accessed_keys(d1) == {'a', 'b1', 'c', 'd1', 'd2'} - d2 = d1.get('b') - assert not_accessed_keys(d1) == {'a', 'b1', 'c', 'd1', 'd2'} + d1["d"][0]["1"] + assert not_accessed_keys(d1) == {"a", "b1", "c", "d1", "d2"} + d2 = d1.get("b") + assert not_accessed_keys(d1) == {"a", "b1", "c", "d1", "d2"} for k, v in d2.iteritems(): pass - assert not_accessed_keys(d1) == {'a', 'c', 'd1', 'd2'} - - d1.get('c') - assert not_accessed_keys(d1) == {'a', 'd1', 'd2'} - - d1 = MementoDict([ - ('a', [1, 2, 3]), - ('b', MementoDict({'1': 1})), - ('c', 1), - ('d', [MementoDict({'1': 1, '2': 2}), MementoDict({'1': 2, '2': 2})]), - ('e', MementoDict([('1', 1), ('1', 2), ('1', 3)])) - ]) - assert not_accessed_keys(d1) == {'a', 'b1', 'c', 'd1', 'd2', 'e1'} + assert not_accessed_keys(d1) == {"a", "c", "d1", "d2"} + + d1.get("c") + assert not_accessed_keys(d1) == {"a", "d1", "d2"} + + d1 = MementoDict( + [ + ("a", [1, 2, 3]), + ("b", MementoDict({"1": 1})), + ("c", 1), + ("d", [MementoDict({"1": 1, "2": 2}), MementoDict({"1": 2, "2": 2})]), + ("e", MementoDict([("1", 1), ("1", 2), ("1", 3)])), + ] + ) + assert not_accessed_keys(d1) == {"a", "b1", "c", "d1", "d2", "e1"} def test_memento_dict(): """Check MementoDict class.""" - d = MementoDict({'a': 1, 'b': 2}) - assert d == {'a': 1, 'b': 2} + d = MementoDict({"a": 1, "b": 2}) + assert d == {"a": 1, "b": 2} - d = MementoDict([('a', 1), ('b', 2)]) - assert d == {'a': 1, 'b': 2} + d = MementoDict([("a", 1), ("b", 2)]) + assert d == {"a": 1, "b": 2} - d = MementoDict([('a', 1), ('a', 2)]) - assert d == {'a': [1, 2]} + d = MementoDict([("a", 1), ("a", 2)]) + assert d == {"a": [1, 2]} # NOTE: is this the expected behavior? - d = MementoDict([('a', 1), ('a', [2, 3])]) - assert d == {'a': [1, 2, 3]} + d = MementoDict([("a", 1), ("a", [2, 3])]) + assert d == {"a": [1, 2, 3]} def test_yaml2json(): """Test yaml to json file conversion""" - root_dir = './tests/fixtures/books/' - source = os.path.join(root_dir, 'ymls/') - destination = os.path.join(root_dir, 'results') + root_dir = "./tests/fixtures/books/" + source = os.path.join(root_dir, "ymls/") + destination = os.path.join(root_dir, "results") yaml2json(source, destination) - with open(os.path.join(root_dir, 'books_title_mock.json')) as s: + with open(os.path.join(root_dir, "books_title_mock.json")) as s: # read the mock JSON file json_mock = json.load(s) assert os.path.exists(destination) - with open(os.path.join(destination, 'books_title.json')) as s: + with open(os.path.join(destination, "books_title.json")) as s: # read the newly created JSON file json_converted = json.load(s) @@ -164,51 +183,55 @@ def test_yaml2json(): volume_params = [ - ('v1', '1', False), - ('v 1', '1', False), - ('v.1', '1', False), - ('v. 1', '1', False), - ('v . 1', '1', False), - ('vol.1', '1', False), - ('Vol. 2', '2', False), - ('voL . 1', None, False), - ('volume.1', '1', False), - ('Volume. 1', '1', False), - ('volume . 3', '3', False), - ('pt 5', '5', False), - ('part 5', '5', False), - ('par 5', None, False), - ('v. A', 'A', False), - ('val. 5', None, False), - ('part III', 'III', False), - ('March 1996', None, False), + ("v1", "1", False), + ("v 1", "1", False), + ("v.1", "1", False), + ("v. 1", "1", False), + ("v . 1", "1", False), + ("vol.1", "1", False), + ("Vol. 2", "2", False), + ("voL . 1", None, False), + ("volume.1", "1", False), + ("Volume. 1", "1", False), + ("volume . 3", "3", False), + ("pt 5", "5", False), + ("part 5", "5", False), + ("par 5", None, False), + ("v. A", "A", False), + ("val. 5", None, False), + ("part III", "III", False), + ("March 1996", None, False), ] -@pytest.mark.parametrize('value, expected, raise_exception', volume_params) +@pytest.mark.parametrize("value, expected, raise_exception", volume_params) def test_extract_volume_number(value, expected, raise_exception): """Test extracting volume number.""" if raise_exception: with pytest.raises(MissingRequiredField): - assert extract_volume_number( - value, - raise_exception=raise_exception - ) == expected + assert ( + extract_volume_number(value, raise_exception=raise_exception) + == expected + ) else: assert extract_volume_number(value) == expected volume_info_params = [ ( - 'print version, paperback ({})'.format(vol_str), - dict(volume=expected, description='print version, paperback') if expected else None + "print version, paperback ({})".format(vol_str), + ( + dict(volume=expected, description="print version, paperback") + if expected + else None + ), ) for vol_str, expected, raise_exception in volume_params ] -volume_info_params.append(('print version, paperback v.5', None)) +volume_info_params.append(("print version, paperback v.5", None)) -@pytest.mark.parametrize('value, expected', volume_info_params) +@pytest.mark.parametrize("value, expected", volume_info_params) def test_extract_volume_info(value, expected): """Test extracting volume info.""" assert extract_volume_info(value) == expected diff --git a/tests/test_videos_project.py b/tests/test_videos_project.py index b10ccdfe..a4d2c293 100644 --- a/tests/test_videos_project.py +++ b/tests/test_videos_project.py @@ -17,7 +17,9 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Video rules tests.""" -import mock + +from unittest import mock + from helpers import load_fixture_file, mock_contributor_fetch, validate from cds_dojson.marc21.fields.videos.utils import language_to_isocode @@ -27,7 +29,7 @@ def test_required_fields(app): """Test required fields.""" - marcxml = load_fixture_file('videos_project.xml') + marcxml = load_fixture_file("videos_project.xml") with app.app_context(): with mock.patch( @@ -39,48 +41,58 @@ def test_required_fields(app): record = model.do(blob) assert record == { - '$schema': { - '$ref': ('https://cds.cern.ch/schemas/' - 'records/videos/project/project-v1.0.0.json') + "$schema": { + "$ref": ( + "https://cds.cern.ch/schemas/" + "records/videos/project/project-v1.0.0.json" + ) }, - '_access': {'update': ['another.user@cern.ch']}, - 'category': 'CERN', - 'contributors': [ - {'name': 'CERN Video Productions', 'role': 'Producer'}, - {'name': 'CERN Video Productions', 'role': 'Director'} + "_access": {"update": ["another.user@cern.ch"]}, + "category": "CERN", + "contributors": [ + {"name": "CERN Video Productions", "role": "Producer"}, + {"name": "CERN Video Productions", "role": "Director"}, + ], + "keywords": [ + {"name": "Higgs", "source": "CERN"}, + {"name": "anniversary", "source": "CERN"}, + ], + "recid": 2272969, + "report_number": ["CERN-MOVIE-2017-023"], + "title": {"title": "Higgs anniversary 5Y"}, + "type": "MOVIE", + "videos": [ + {"$ref": "https://cds.cern.ch/record/1"}, + {"$ref": "https://cds.cern.ch/record/2"}, ], - 'keywords': [{'name': 'Higgs', 'source': 'CERN'}, - {'name': 'anniversary', 'source': 'CERN'}], - 'recid': 2272969, - 'report_number': ['CERN-MOVIE-2017-023'], - 'title': {'title': 'Higgs anniversary 5Y'}, - 'type': 'MOVIE', - 'videos': [{'$ref': 'https://cds.cern.ch/record/1'}, - {'$ref': 'https://cds.cern.ch/record/2'}], - 'external_system_identifiers': [ - {'schema': 'AVW', 'value': 'AVW.project.2963'} + "external_system_identifiers": [ + {"schema": "AVW", "value": "AVW.project.2963"} ], - 'modified_by': 'another.user@cern.ch', + "modified_by": "another.user@cern.ch", } # Add required fields calculated by post-process tasks. - record['publication_date'] = '2017-07-04' - record['date'] = '2017-07-04' + record["publication_date"] = "2017-07-04" + record["date"] = "2017-07-04" validate(record) def test_fields(app): """Test fields.""" - marcxml = ("""""" - """{0}""") + marcxml = ( + """""" + """{0}""" + ) def check_transformation(marcxml_body, json_body): blob = create_record(marcxml.format(marcxml_body)) record = model.do(blob) expected = { - '$schema': { - '$ref': ('https://cds.cern.ch/schemas/' - 'records/videos/project/project-v1.0.0.json') + "$schema": { + "$ref": ( + "https://cds.cern.ch/schemas/" + "records/videos/project/project-v1.0.0.json" + ) } } expected.update(**json_body) @@ -99,8 +111,9 @@ def check_transformation(marcxml_body, json_body): 2001-02-03 - """, { - }) + """, + {}, + ) check_transformation( """ @@ -108,33 +121,35 @@ def check_transformation(marcxml_body, json_body): test2 test3 - """, { - 'license': [ - {'material': 'test1', 'url': 'test2', 'license': 'test3'} + """, + { + "license": [ + {"material": "test1", "url": "test2", "license": "test3"} ], - }) + }, + ) check_transformation( """ test1 test2 - """, { - 'keywords': [ - {'name': 'test1', 'source': 'test2'} - ], - }) + """, + { + "keywords": [{"name": "test1", "source": "test2"}], + }, + ) check_transformation( """ test1 test2 - """, { - 'related_links': [ - {'name': 'test1', 'url': 'test2'} - ], - }) + """, + { + "related_links": [{"name": "test1", "url": "test2"}], + }, + ) check_transformation( """ @@ -143,13 +158,17 @@ def check_transformation(marcxml_body, json_body): test2 - """, { - 'translations': [{ - 'title': {'title': 'test1'}, - 'description': 'test2', - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "title": {"title": "test1"}, + "description": "test2", + "language": "fr", + } + ] + }, + ) check_transformation( """ @@ -158,35 +177,47 @@ def check_transformation(marcxml_body, json_body): test2 - """, { - 'translations': [{ - 'title': {'title': 'test1'}, - 'description': 'test2', - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "title": {"title": "test1"}, + "description": "test2", + "language": "fr", + } + ] + }, + ) check_transformation( """ test1 - """, { - 'translations': [{ - 'title': {'title': 'test1'}, - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "title": {"title": "test1"}, + "language": "fr", + } + ] + }, + ) check_transformation( """ test2 - """, { - 'translations': [{ - 'description': 'test2', - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "description": "test2", + "language": "fr", + } + ] + }, + ) check_transformation( """ @@ -210,25 +241,33 @@ def check_transformation(marcxml_body, json_body): Test User Director - """, { - 'contributors': [ - {'name': 'CERN Video Productions', 'role': 'Producer'}, - {'name': 'CERN Video Productions', 'role': 'Director'}, + """, + { + "contributors": [ + {"name": "CERN Video Productions", "role": "Producer"}, + {"name": "CERN Video Productions", "role": "Director"}, { - 'email': u'tuser2@cern.ch', - 'ids': [{'source': 'CERN', 'value': u'9876542'}, - {'source': 'CDS', 'value': u'2123456789'}], - 'name': 'User, Test 2', 'role': 'Camera Operator' + "email": "tuser2@cern.ch", + "ids": [ + {"source": "CERN", "value": "9876542"}, + {"source": "CDS", "value": "2123456789"}, + ], + "name": "User, Test 2", + "role": "Camera Operator", }, - {'name': 'test2', 'role': 'Credits'}, + {"name": "test2", "role": "Credits"}, { - 'email': u'tuser@cern.ch', - 'ids': [{'source': 'CERN', 'value': u'987654'}, - {'source': 'CDS', 'value': u'123456789'}], - 'name': 'User, Test', 'role': 'Director' + "email": "tuser@cern.ch", + "ids": [ + {"source": "CERN", "value": "987654"}, + {"source": "CDS", "value": "123456789"}, + ], + "name": "User, Test", + "role": "Director", }, ] - }) + }, + ) check_transformation( """ @@ -237,28 +276,34 @@ def check_transformation(marcxml_body, json_body): test2 - """, { - 'videos': [ - {'$ref': 'test1'}, - {'$ref': 'test2'}, + """, + { + "videos": [ + {"$ref": "test1"}, + {"$ref": "test2"}, ] - }) + }, + ) check_transformation( """ test1 - """, { - 'description': 'test1', - }) + """, + { + "description": "test1", + }, + ) check_transformation( """ test1 - """, { - 'note': 'test1', - }) + """, + { + "note": "test1", + }, + ) check_transformation( """ @@ -267,19 +312,21 @@ def check_transformation(marcxml_body, json_body): FCS.project.2345 - """, { - 'external_system_identifiers': [ - {'value': '000012345.MDD', 'schema': 'ALEPH'}, - {'value': 'FCS.project.2345', 'schema': 'FCS'} + """, + { + "external_system_identifiers": [ + {"value": "000012345.MDD", "schema": "ALEPH"}, + {"value": "FCS.project.2345", "schema": "FCS"}, ] - }) + }, + ) def test_language_to_isocode(): """Test language to isocode.""" - assert language_to_isocode('eng') == 'en' - assert language_to_isocode('eng-fre') == 'en-fr' - assert language_to_isocode('ITA') == 'it' - assert language_to_isocode('silent') == 'silent' - assert language_to_isocode('sil') == 'silent' - assert language_to_isocode('fuu') == 'silent' + assert language_to_isocode("eng") == "en" + assert language_to_isocode("eng-fre") == "en-fr" + assert language_to_isocode("ITA") == "it" + assert language_to_isocode("silent") == "silent" + assert language_to_isocode("sil") == "silent" + assert language_to_isocode("fuu") == "silent" diff --git a/tests/test_videos_video.py b/tests/test_videos_video.py index e1ad2362..ba89c725 100644 --- a/tests/test_videos_video.py +++ b/tests/test_videos_video.py @@ -17,7 +17,9 @@ # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Video rules tests.""" -import mock + +from unittest import mock + from helpers import load_fixture_file, mock_contributor_fetch, validate from cds_dojson.marc21.fields.videos.utils import language_to_isocode @@ -27,248 +29,306 @@ def test_required_fields(app): """Test required fields.""" - marcxml = load_fixture_file('videos_video.xml') + marcxml = load_fixture_file("videos_video.xml") with app.app_context(): with mock.patch( "cds_dojson.marc21.fields.utils.get_author_info_from_people_collection", - side_effect=lambda contributor: mock_contributor_fetch(contributor) + side_effect=lambda contributor: mock_contributor_fetch(contributor), ): blob = create_record(marcxml) record = model.do(blob) expected = { - '$schema': { - '$ref': ('https://cds.cern.ch/schemas/records/videos/video/' - 'video-v1.0.0.json') + "$schema": { + "$ref": ( + "https://cds.cern.ch/schemas/records/videos/video/" + "video-v1.0.0.json" + ) }, - '_access': {'read': ['test-group@cern.ch', - 'cds-admin@cern.ch', - 'test-email@cern.ch', - 'example@test.com'], - 'update': ['another.user@cern.ch', - 'tuser@cern.ch']}, - '_files': [ + "_access": { + "read": [ + "test-group@cern.ch", + "cds-admin@cern.ch", + "test-email@cern.ch", + "example@test.com", + ], + "update": ["another.user@cern.ch", "tuser@cern.ch"], + }, + "_files": [ { - 'filepath': 'MediaArchive/Video/Masters/Movies/CERN/2017/CERN-MOVIE-2017-023/Final_Output/CERN-MOVIE-2017-023-001.mov', - 'key': 'CERN-MOVIE-2017-023-001.mov', - 'tags': { - 'media_type': 'video', - 'content_type': 'mov', - 'context_type': 'master', - 'preview': True, + "filepath": "MediaArchive/Video/Masters/Movies/CERN/2017/CERN-MOVIE-2017-023/Final_Output/CERN-MOVIE-2017-023-001.mov", + "key": "CERN-MOVIE-2017-023-001.mov", + "tags": { + "media_type": "video", + "content_type": "mov", + "context_type": "master", + "preview": True, }, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-5872-kbps-1920x1080-audio-128-kbps-stereo.mp4', - 'key': 'CERN-MOVIE-2017-023-001-5872-kbps-1920x1080-audio-128-kbps-stereo.mp4', - 'tags_to_guess_preset': {'preset': '1080p', 'video_bitrate': 5872}, - 'tags': { - 'media_type': 'video', - 'content_type': 'mp4', - 'context_type': 'subformat', + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-5872-kbps-1920x1080-audio-128-kbps-stereo.mp4", + "key": "CERN-MOVIE-2017-023-001-5872-kbps-1920x1080-audio-128-kbps-stereo.mp4", + "tags_to_guess_preset": { + "preset": "1080p", + "video_bitrate": 5872, + }, + "tags": { + "media_type": "video", + "content_type": "mp4", + "context_type": "subformat", }, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-2672-kbps-1280x720-audio-128-kbps-stereo.mp4', - 'key': 'CERN-MOVIE-2017-023-001-2672-kbps-1280x720-audio-128-kbps-stereo.mp4', - 'tags_to_guess_preset': {'preset': '720p', 'video_bitrate': 2672}, - 'tags': { - 'media_type': 'video', - 'content_type': 'mp4', - 'context_type': 'subformat', + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-2672-kbps-1280x720-audio-128-kbps-stereo.mp4", + "key": "CERN-MOVIE-2017-023-001-2672-kbps-1280x720-audio-128-kbps-stereo.mp4", + "tags_to_guess_preset": { + "preset": "720p", + "video_bitrate": 2672, + }, + "tags": { + "media_type": "video", + "content_type": "mp4", + "context_type": "subformat", }, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-1436-kbps-853x480-audio-64-kbps-stereo.mp4', - 'key': 'CERN-MOVIE-2017-023-001-1436-kbps-853x480-audio-64-kbps-stereo.mp4', - 'tags_to_guess_preset': {'preset': '480p', 'video_bitrate': 1436}, - 'tags': { - 'media_type': 'video', - 'content_type': 'mp4', - 'context_type': 'subformat', + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-1436-kbps-853x480-audio-64-kbps-stereo.mp4", + "key": "CERN-MOVIE-2017-023-001-1436-kbps-853x480-audio-64-kbps-stereo.mp4", + "tags_to_guess_preset": { + "preset": "480p", + "video_bitrate": 1436, + }, + "tags": { + "media_type": "video", + "content_type": "mp4", + "context_type": "subformat", }, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-836-kbps-640x360-audio-64-kbps-stereo.mp4', - 'key': 'CERN-MOVIE-2017-023-001-836-kbps-640x360-audio-64-kbps-stereo.mp4', - 'tags_to_guess_preset': {'preset': '360p', 'video_bitrate': 836}, - 'tags': { - 'media_type': 'video', - 'content_type': 'mp4', - 'context_type': 'subformat', + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-836-kbps-640x360-audio-64-kbps-stereo.mp4", + "key": "CERN-MOVIE-2017-023-001-836-kbps-640x360-audio-64-kbps-stereo.mp4", + "tags_to_guess_preset": { + "preset": "360p", + "video_bitrate": 836, + }, + "tags": { + "media_type": "video", + "content_type": "mp4", + "context_type": "subformat", }, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-386-kbps-426x240-audio-64-kbps-stereo.mp4', - 'key': 'CERN-MOVIE-2017-023-001-386-kbps-426x240-audio-64-kbps-stereo.mp4', - 'tags_to_guess_preset': {'preset': '240p', 'video_bitrate': 386}, - 'tags': { - 'media_type': 'video', - 'content_type': 'mp4', - 'context_type': 'subformat', + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-386-kbps-426x240-audio-64-kbps-stereo.mp4", + "key": "CERN-MOVIE-2017-023-001-386-kbps-426x240-audio-64-kbps-stereo.mp4", + "tags_to_guess_preset": { + "preset": "240p", + "video_bitrate": 386, + }, + "tags": { + "media_type": "video", + "content_type": "mp4", + "context_type": "subformat", }, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-5-percent.jpg', - 'key': 'frame-1.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 5} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-5-percent.jpg", + "key": "frame-1.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 5}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-5-percent.jpg', - 'key': 'posterframe.jpg', - 'tags': { - 'media_type': 'image', - 'height': '360', - 'width': '640', - 'content_type': 'jpg', - 'context_type': 'poster', + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-5-percent.jpg", + "key": "posterframe.jpg", + "tags": { + "media_type": "image", + "height": "360", + "width": "640", + "content_type": "jpg", + "context_type": "poster", }, - 'tags_to_transform': {'timestamp': 5}, + "tags_to_transform": {"timestamp": 5}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-15-percent.jpg', - 'key': 'frame-2.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 15} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-15-percent.jpg", + "key": "frame-2.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 15}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-25-percent.jpg', - 'key': 'frame-3.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 25} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-25-percent.jpg", + "key": "frame-3.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 25}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-35-percent.jpg', - 'key': 'frame-4.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 35} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-35-percent.jpg", + "key": "frame-4.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 35}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-45-percent.jpg', - 'key': 'frame-5.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 45} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-45-percent.jpg", + "key": "frame-5.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 45}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-55-percent.jpg', - 'key': 'frame-6.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 55} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-55-percent.jpg", + "key": "frame-6.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 55}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-65-percent.jpg', - 'key': 'frame-7.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 65} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-65-percent.jpg", + "key": "frame-7.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 65}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-75-percent.jpg', - 'key': 'frame-8.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 75} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-75-percent.jpg", + "key": "frame-8.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 75}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-85-percent.jpg', - 'key': 'frame-9.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 85} + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-85-percent.jpg", + "key": "frame-9.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 85}, }, { - 'filepath': 'MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-95-percent.jpg', - 'key': 'frame-10.jpg', - 'tags': {'content_type': 'jpg', - 'context_type': 'frame', - 'media_type': 'image'}, - 'tags_to_transform': {'timestamp': 95} - } + "filepath": "MediaArchive/Video/Public/Movies/CERN/2017/CERN-MOVIE-2017-023/CERN-MOVIE-2017-023-001/CERN-MOVIE-2017-023-001-posterframe-640x360-at-95-percent.jpg", + "key": "frame-10.jpg", + "tags": { + "content_type": "jpg", + "context_type": "frame", + "media_type": "image", + }, + "tags_to_transform": {"timestamp": 95}, + }, ], - '_project_id': 'https://cds.cern.ch/record/1', - 'category': 'CERN', - 'contributors': [ - {'name': 'CERN Video Productions', 'role': 'Producer'}, - {'name': 'CERN Video Productions', 'role': 'Director'}, - {'email': 'tuser@cern.ch', - 'ids': [{'source': 'CERN', 'value': '987654'}, - {'source': 'CDS', 'value': '123456789'}], - 'name': 'User, Test', 'role': 'Director' + "_project_id": "https://cds.cern.ch/record/1", + "category": "CERN", + "contributors": [ + {"name": "CERN Video Productions", "role": "Producer"}, + {"name": "CERN Video Productions", "role": "Director"}, + { + "email": "tuser@cern.ch", + "ids": [ + {"source": "CERN", "value": "987654"}, + {"source": "CDS", "value": "123456789"}, + ], + "name": "User, Test", + "role": "Director", }, { - 'email': u'tuser2@cern.ch', - 'ids': [{'source': 'CERN', 'value': u'9876542'}, - {'source': 'CDS', 'value': u'2123456789'}], - 'name': 'User, Test 2', 'role': 'Director' + "email": "tuser2@cern.ch", + "ids": [ + {"source": "CERN", "value": "9876542"}, + {"source": "CDS", "value": "2123456789"}, + ], + "name": "User, Test 2", + "role": "Director", }, { - 'email': u'tuser@cern.ch', - 'ids': [{'source': 'CERN', 'value': u'987654'}, - {'source': 'CDS', 'value': u'123456789'}], - 'name': 'User, Test', 'role': 'Editor' + "email": "tuser@cern.ch", + "ids": [ + {"source": "CERN", "value": "987654"}, + {"source": "CDS", "value": "123456789"}, + ], + "name": "User, Test", + "role": "Editor", }, ], - 'copyright': {'holder': 'CERN', 'year': '2017'}, - 'date': '2017-07-04', - 'description': ('Where were you on 4 July 2012, the day in which ' - 'the Higgs boson discovery was announced?'), - 'duration': '00:01:09', - 'keywords': [ - {'name': 'higgs', 'source': 'CERN'}, - {'name': 'anniversary', 'source': 'CERN'} + "copyright": {"holder": "CERN", "year": "2017"}, + "date": "2017-07-04", + "description": ( + "Where were you on 4 July 2012, the day in which " + "the Higgs boson discovery was announced?" + ), + "duration": "00:01:09", + "keywords": [ + {"name": "higgs", "source": "CERN"}, + {"name": "anniversary", "source": "CERN"}, ], - 'language': u'en', - 'recid': 2272973, - 'report_number': ['CERN-MOVIE-2017-023-001'], - 'title': {'title': 'Happy 5th anniversary, Higgs boson!'}, - 'type': 'MOVIE', - 'external_system_identifiers': [ - {'schema': 'AVW', 'value': 'AVW.clip.3447'} + "language": "en", + "recid": 2272973, + "report_number": ["CERN-MOVIE-2017-023-001"], + "title": {"title": "Happy 5th anniversary, Higgs boson!"}, + "type": "MOVIE", + "external_system_identifiers": [ + {"schema": "AVW", "value": "AVW.clip.3447"} ], - 'modified_by': 'another.user@cern.ch', + "modified_by": "another.user@cern.ch", } assert record == expected # Add required fields calculated by post-process tasks. - record['publication_date'] = '2017-07-04' - record['doi'] = 'CERN/2272973' - record['license'] = [ - {'license': 'CC BY 4.0', - 'url': 'https://creativecommons.org/licenses/by/4.0/'} + record["publication_date"] = "2017-07-04" + record["doi"] = "CERN/2272973" + record["license"] = [ + { + "license": "CC BY 4.0", + "url": "https://creativecommons.org/licenses/by/4.0/", + } ] validate(record) def test_fields(app): """Test fields.""" - marcxml = ("""""" - """{0}""") + marcxml = ( + """""" + """{0}""" + ) def check_transformation(marcxml_body, json_body): blob = create_record(marcxml.format(marcxml_body)) record = model.do(blob) expected = { - '$schema': { - '$ref': ('https://cds.cern.ch/schemas/records/videos/video/' - 'video-v1.0.0.json') + "$schema": { + "$ref": ( + "https://cds.cern.ch/schemas/records/videos/video/" + "video-v1.0.0.json" + ) } } expected.update(**json_body) @@ -284,33 +344,41 @@ def check_transformation(marcxml_body, json_body): test1 - """, { - 'original_source': 'test1', - }) + """, + { + "original_source": "test1", + }, + ) check_transformation( """ test1 - """, { - 'note': 'test1', - }) + """, + { + "note": "test1", + }, + ) check_transformation( """ test1 - """, { - 'note': 'test1', - }) + """, + { + "note": "test1", + }, + ) check_transformation( """ 2017-03-15 - """, { - 'date': '2017-03-15', - }) + """, + { + "date": "2017-03-15", + }, + ) check_transformation( """ @@ -320,16 +388,18 @@ def check_transformation(marcxml_body, json_body): loc_test1A - """, { - 'physical_medium': [ + """, + { + "physical_medium": [ { - 'camera': 'test2A', - 'medium_standard': 'test1A', - 'location': 'loc_test1A', - 'sequence_number': [], + "camera": "test2A", + "medium_standard": "test1A", + "location": "loc_test1A", + "sequence_number": [], } ] - }) + }, + ) check_transformation( """ @@ -350,28 +420,32 @@ def check_transformation(marcxml_body, json_body): A loc_test1A - """, { - 'physical_medium': [ + """, + { + "physical_medium": [ { - 'sequence_number': ['A'], - 'camera': 'test2A', - 'medium_standard': 'test1A', - 'location': 'loc_test1A' + "sequence_number": ["A"], + "camera": "test2A", + "medium_standard": "test1A", + "location": "loc_test1A", }, { - 'sequence_number': ['B'], - 'camera': 'test2B', - 'medium_standard': 'test1B', - 'location': 'loc_test1B' - } + "sequence_number": ["B"], + "camera": "test2B", + "medium_standard": "test1B", + "location": "loc_test1B", + }, ] - }) + }, + ) check_transformation( """ test1 - """, {'note': 'test1'}) + """, + {"note": "test1"}, + ) check_transformation( """ @@ -379,11 +453,13 @@ def check_transformation(marcxml_body, json_body): test2 test3 - """, { - 'license': [ - {'material': 'test1', 'url': 'test2', 'license': 'test3'} + """, + { + "license": [ + {"material": "test1", "url": "test2", "license": "test3"} ], - }) + }, + ) check_transformation( """ @@ -391,22 +467,26 @@ def check_transformation(marcxml_body, json_body): test2 test3 - """, { - 'copyright': { - 'holder': 'test1', 'year': 'test2', 'message': 'test3' + """, + { + "copyright": { + "holder": "test1", + "year": "test2", + "message": "test3", }, - }) + }, + ) check_transformation( """ test1 test2 - """, { - 'keywords': [ - {'name': 'test1', 'source': 'test2'} - ], - }) + """, + { + "keywords": [{"name": "test1", "source": "test2"}], + }, + ) check_transformation( """ @@ -416,31 +496,33 @@ def check_transformation(marcxml_body, json_body): test1 test2 - """, { - '_project_id': 'test1', - 'related_links': [ - {'name': 'test1', 'url': 'test2'} - ], - }) + """, + { + "_project_id": "test1", + "related_links": [{"name": "test1", "url": "test2"}], + }, + ) check_transformation( """ test1 - """, { - '_project_id': 'test1', - }) + """, + { + "_project_id": "test1", + }, + ) check_transformation( """ test1 test2 - """, { - 'related_links': [ - {'name': 'test1', 'url': 'test2'} - ], - }) + """, + { + "related_links": [{"name": "test1", "url": "test2"}], + }, + ) check_transformation( """ @@ -457,20 +539,29 @@ def check_transformation(marcxml_body, json_body): CERN-MOVIE-1965-001 https://cds.cern.ch/api/record/2017733 - """, {'_project_id': 'https://cds.cern.ch/api/record/2017733', - 'related_links': [ - {'name': 'Version anglaise', 'url': 'http://cds.cern.ch/record/43172'}, - {'name': 'Version allemande', 'url': 'https://cds.cern.ch/record/2194933'}, - ]} + """, + { + "_project_id": "https://cds.cern.ch/api/record/2017733", + "related_links": [ + { + "name": "Version anglaise", + "url": "http://cds.cern.ch/record/43172", + }, + { + "name": "Version allemande", + "url": "https://cds.cern.ch/record/2194933", + }, + ], + }, ) check_transformation( """ test1 - """, { - 'location': 'test1' - }) + """, + {"location": "test1"}, + ) check_transformation( """ @@ -479,13 +570,17 @@ def check_transformation(marcxml_body, json_body): test2 - """, { - 'translations': [{ - 'title': {'title': 'test1'}, - 'description': 'test2', - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "title": {"title": "test1"}, + "description": "test2", + "language": "fr", + } + ] + }, + ) check_transformation( """ @@ -494,62 +589,74 @@ def check_transformation(marcxml_body, json_body): test2 - """, { - 'translations': [{ - 'title': {'title': 'test1'}, - 'description': 'test2', - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "title": {"title": "test1"}, + "description": "test2", + "language": "fr", + } + ] + }, + ) check_transformation( """ test1 - """, { - 'translations': [{ - 'title': {'title': 'test1'}, - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "title": {"title": "test1"}, + "language": "fr", + } + ] + }, + ) check_transformation( """ 0:0 16:9 - """, { - 'duration': '00:00:00' - }) + """, + {"duration": "00:00:00"}, + ) check_transformation( """ 12:33:12 16:9 - """, { - 'duration': '12:33:12' - }) + """, + {"duration": "12:33:12"}, + ) check_transformation( """ 16:9, 16:9 - """, { - 'duration': '00:00:00' - }) + """, + {"duration": "00:00:00"}, + ) check_transformation( """ test2 - """, { - 'translations': [{ - 'description': 'test2', - 'language': 'fr', - }] - }) + """, + { + "translations": [ + { + "description": "test2", + "language": "fr", + } + ] + }, + ) check_transformation( """ @@ -579,57 +686,72 @@ def check_transformation(marcxml_body, json_body): Jessica, Jones Produced by - """, { - 'contributors': [ - {'name': 'CERN Video Productions', 'role': 'Producer'}, - {'name': 'CERN Video Productions', 'role': 'Director'}, + """, + { + "contributors": [ + {"name": "CERN Video Productions", "role": "Producer"}, + {"name": "CERN Video Productions", "role": "Director"}, { - 'email': u'tuser2@cern.ch', - 'ids': [{'source': 'CERN', 'value': u'9876542'}, - {'source': 'CDS', 'value': u'2123456789'}], - 'name': 'User, Test 2', 'role': 'Camera Operator' + "email": "tuser2@cern.ch", + "ids": [ + {"source": "CERN", "value": "9876542"}, + {"source": "CDS", "value": "2123456789"}, + ], + "name": "User, Test 2", + "role": "Camera Operator", }, - {'name': 'test2', 'role': 'Credits'}, + {"name": "test2", "role": "Credits"}, { - 'email': u'tuser@cern.ch', - 'ids': [{'source': 'CERN', 'value': u'987654'}, - {'source': 'CDS', 'value': u'123456789'}], - 'name': 'User, Test', 'role': 'Director' + "email": "tuser@cern.ch", + "ids": [ + {"source": "CERN", "value": "987654"}, + {"source": "CDS", "value": "123456789"}, + ], + "name": "User, Test", + "role": "Director", }, { - 'role': 'Producer', - 'ids': [{'source': 'CDS', 'value': "CDS"}], - 'name': 'Jessica, Jones' + "role": "Producer", + "ids": [{"source": "CDS", "value": "CDS"}], + "name": "Jessica, Jones", }, ] - }) + }, + ) check_transformation( """ CERN video productions Script and Director - """, { - 'contributors': [ - {'name': 'CERN Video Productions', 'role': 'Screenwriter'}, - {'name': 'CERN Video Productions', 'role': 'Director'} - ]}) + """, + { + "contributors": [ + {"name": "CERN Video Productions", "role": "Screenwriter"}, + {"name": "CERN Video Productions", "role": "Director"}, + ] + }, + ) check_transformation( """ Jean-Claude Vialis Réalisation - """, { - 'contributors': [ - {'name': 'Jean-Claude Vialis', 'role': 'Producer'}, - ]}) + """, + { + "contributors": [ + {"name": "Jean-Claude Vialis", "role": "Producer"}, + ] + }, + ) check_transformation( """ test1 - """, {'internal_note': 'test1'} + """, + {"internal_note": "test1"}, ) check_transformation( """ @@ -639,7 +761,8 @@ def check_transformation(marcxml_body, json_body): test2 - """, {'internal_note': 'test1\ntest2'} + """, + {"internal_note": "test1\ntest2"}, ) check_transformation( """ @@ -647,7 +770,8 @@ def check_transformation(marcxml_body, json_body): CERN50 AcceleratorsDetectors - """, {'internal_categories': {'CERN50': ['AcceleratorsDetectors']}} + """, + {"internal_categories": {"CERN50": ["AcceleratorsDetectors"]}}, ) check_transformation( """ @@ -658,8 +782,11 @@ def check_transformation(marcxml_body, json_body): CERN50 AcceleratorsDetectors - """, {'internal_note': 'test1', - 'internal_categories': {'CERN50': ['AcceleratorsDetectors']}} + """, + { + "internal_note": "test1", + "internal_categories": {"CERN50": ["AcceleratorsDetectors"]}, + }, ) check_transformation( """ @@ -671,12 +798,14 @@ def check_transformation(marcxml_body, json_body): test3 test4 - """, { - 'subject': { - 'source': 'test2', - 'term': 'test1', + """, + { + "subject": { + "source": "test2", + "term": "test1", } - }) + }, + ) check_transformation( """ @@ -686,24 +815,25 @@ def check_transformation(marcxml_body, json_body): test4 test5 - """, { - 'accelerator_experiment': { - 'accelerator': 'test1', - 'experiment': 'test2', - 'study': 'test3', - 'facility': 'test4', - 'project': 'test5', + """, + { + "accelerator_experiment": { + "accelerator": "test1", + "experiment": "test2", + "study": "test3", + "facility": "test4", + "project": "test5", } - } + }, ) def test_language_to_isocode(): """Test language to isocode.""" - assert language_to_isocode('eng') == 'en' - assert language_to_isocode('eng-fre') == 'en-fr' - assert language_to_isocode('ITA') == 'it' - assert language_to_isocode('silent') == 'silent' - assert language_to_isocode('sil') == 'silent' - assert language_to_isocode('fuu') == 'silent' - assert language_to_isocode('French') == 'fr' + assert language_to_isocode("eng") == "en" + assert language_to_isocode("eng-fre") == "en-fr" + assert language_to_isocode("ITA") == "it" + assert language_to_isocode("silent") == "silent" + assert language_to_isocode("sil") == "silent" + assert language_to_isocode("fuu") == "silent" + assert language_to_isocode("French") == "fr"