diff --git a/app/models/sqs_message.rb b/app/models/sqs_message.rb index 55011406..0baf6c70 100644 --- a/app/models/sqs_message.rb +++ b/app/models/sqs_message.rb @@ -6,7 +6,7 @@ class SqsMessage def initialize(thesis) @thesis = thesis @package_id = "etd_#{@thesis.id}" - @metadata_uri = thesis.dspace_metadata.blob.url(expires_in: 604800) + @metadata_uri = thesis.dspace_metadata.blob.url(expires_in: 604_800) end def message_attributes @@ -35,8 +35,8 @@ def map_files .sort_by { |item| ordered_filter.index(item[:purpose]) } .map do |f| { - 'BitstreamName' => f.blob.filename.to_s, - 'FileLocation' => f.blob.url(expires_in: 604800), + 'BitstreamName' => sanitize_filename_for_dspace(f.blob.filename.to_s), + 'FileLocation' => f.blob.url(expires_in: 604_800), 'BitstreamDescription' => bitstream_description(f) } end @@ -47,7 +47,7 @@ def map_files def collection_handle if @thesis.degrees.any? { |d| d.degree_type.name == 'Doctoral' } ENV.fetch('DSPACE_DOCTORAL_HANDLE') - elsif @thesis.degrees.any? { |d| d.degree_type.name == 'Master' || d.degree_type.name == 'Engineer' } + elsif @thesis.degrees.any? { |d| %w[Master Engineer].include?(d.degree_type.name) } ENV.fetch('DSPACE_GRADUATE_HANDLE') else ENV.fetch('DSPACE_UNDERGRADUATE_HANDLE') @@ -60,4 +60,27 @@ def bitstream_description(file) translated_purpose = file_purposes[file.purpose] "#{translated_purpose} #{file.description}".strip end + + private + + # DSpace chokes on certain unicode characters. If a thesis has files that contain one of these + # characters, DSpace will allow publication, but it won't allow end users to download the files. + # + # This method normalizes the current known list of problematic characters: + # - decomposed combining diacritics + # - zero-width spaces (and presumably directional control characters, which are also invisible) + # - en-dashes (and presumably other dash variants) + # + # As we learn of other problematic characters, we should add them to this method. + def sanitize_filename_for_dspace(filename) + # Normalize to NFC (precomposed form) to fix decomposed combining marks (e.g., í as i + acute + # accent) + normalized = filename.unicode_normalize(:nfc) + + # Strip zero-width and directional control characters + normalized = normalized.gsub(/[\u200B\u200C\u200D\u200E\u200F\u202A-\u202E]/, '') + + # Replace em-dashes, en-dashes, and other dash variants with standard hyphen + normalized.gsub(/[\u2010-\u2015]/, '-') + end end diff --git a/test/models/sqs_message_test.rb b/test/models/sqs_message_test.rb index 6f6f8b03..81344094 100644 --- a/test/models/sqs_message_test.rb +++ b/test/models/sqs_message_test.rb @@ -50,10 +50,12 @@ def dss_friendly_thesis(thesis) @thesis.save @thesis.reload assert_equal 5, @thesis.files.length - assert_equal ['thesis_pdf', 'proquest_form', 'signature_page', 'thesis_source', 'thesis_supplementary_file'], @thesis.files.map{|f| f.purpose} + assert_equal(%w[thesis_pdf proquest_form signature_page thesis_source thesis_supplementary_file], @thesis.files.map do |f| + f.purpose + end) files = SqsMessage.new(@thesis).map_files assert_equal 2, files.length - assert_equal ['Thesis PDF My thesis', 'Supplementary file'], files.map{|f| f['BitstreamDescription']} + assert_equal(['Thesis PDF My thesis', 'Supplementary file'], files.map { |f| f['BitstreamDescription'] }) end test 'thesis_pdf are attached before supplementary files' do @@ -67,10 +69,14 @@ def dss_friendly_thesis(thesis) @thesis.files.last.purpose = 'thesis_supplementary_file' @thesis.save @thesis.reload - assert_equal ['thesis_supplementary_file', 'thesis_pdf', 'thesis_supplementary_file'], @thesis.files.map{|f| f.purpose} + assert_equal(%w[thesis_supplementary_file thesis_pdf thesis_supplementary_file], @thesis.files.map do |f| + f.purpose + end) files = SqsMessage.new(@thesis).map_files - assert_equal ['Thesis PDF', 'Supplementary file', 'Supplementary file'], files.map{|f| f['BitstreamDescription']} - assert_equal ['thesis_pdf.pdf', 'supplemental_file.pdf', 'flexible_pdf.pdf'], files.map{ |f| f['BitstreamName'] } + assert_equal(['Thesis PDF', 'Supplementary file', 'Supplementary file'], files.map do |f| + f['BitstreamDescription'] + end) + assert_equal(['thesis_pdf.pdf', 'supplemental_file.pdf', 'flexible_pdf.pdf'], files.map { |f| f['BitstreamName'] }) # Swapping file purposes will result in the same set of files being sorted into a different order. This is meant to # demonstrate confidence that alphabetical order is not part of the logic being used - the thesis pdf comes first, # followed by supplemental files in the order they were attached. @@ -80,8 +86,10 @@ def dss_friendly_thesis(thesis) @thesis.files.last.purpose = 'thesis_pdf' # last-attached "flexible_pdf" should now be sorted first @thesis.files.second.purpose = 'thesis_supplementary_file' # second-attached "thesis_pdf" should now be sorted last files = SqsMessage.new(@thesis).map_files - assert_equal ['Thesis PDF', 'Supplementary file', 'Supplementary file'], files.map{|f| f['BitstreamDescription']} - assert_equal ['flexible_pdf.pdf', 'supplemental_file.pdf', 'thesis_pdf.pdf'], files.map{ |f| f['BitstreamName'] } + assert_equal(['Thesis PDF', 'Supplementary file', 'Supplementary file'], files.map do |f| + f['BitstreamDescription'] + end) + assert_equal(['flexible_pdf.pdf', 'supplemental_file.pdf', 'thesis_pdf.pdf'], files.map { |f| f['BitstreamName'] }) end test 'returns correct bitstream description' do @@ -153,4 +161,63 @@ def dss_friendly_thesis(thesis) # Not checking the full URI here because ActiveStorage::SetCurrent doesn't generate URIs consistently. assert body_json['MetadataLocation'].ends_with?('some_file.json') end + + test 'sanitize_filename_for_dspace normalizes decomposed unicode to precomposed' do + # Decomposed form: í as i (U+0069) + combining acute accent (U+0301) + decomposed = 'saldías_belen_thesis.pdf' # Will be NFD if created on macOS + sqs = SqsMessage.new(@thesis) + sanitized = sqs.send(:sanitize_filename_for_dspace, decomposed) + + # Should normalize to precomposed form + assert_equal 'saldías_belen_thesis.pdf'.unicode_normalize(:nfc), sanitized + end + + test 'sanitize_filename_for_dspace removes zero-width spaces' do + # Contains U+200B (zero-width space) + filename_with_zwsp = "Liang-thesis\u200b.pdf" + sqs = SqsMessage.new(@thesis) + sanitized = sqs.send(:sanitize_filename_for_dspace, filename_with_zwsp) + assert_equal 'Liang-thesis.pdf', sanitized + end + + test 'sanitize_filename_for_dspace replaces en-dashes with hyphens' do + # U+2013 is en-dash + filename_with_endash = "Siddiqui\u2013sameed-thesis.pdf" + sqs = SqsMessage.new(@thesis) + sanitized = sqs.send(:sanitize_filename_for_dspace, filename_with_endash) + assert_equal 'Siddiqui-sameed-thesis.pdf', sanitized + end + + test 'sanitize_filename_for_dspace preserves safe precomposed accented characters' do + # These are precomposed forms that DSpace accepts + safe_filenames = [ + 'garcía_thesis.pdf', # U+00ED precomposed í + 'strømstad_thesis.pdf', # U+00F8 precomposed ø + 'MillánBarea_thesis.pdf' # U+00E1 precomposed á + ] + sqs = SqsMessage.new(@thesis) + safe_filenames.each do |filename| + sanitized = sqs.send(:sanitize_filename_for_dspace, filename) + assert_equal filename, sanitized, "Safe character filename was modified: #{filename}" + end + end + + test 'sanitize_filename_for_dspace applied to map_files output' do + # Test end-to-end: verify sanitized filenames appear in map_files output + f = Rails.root.join('test', 'fixtures', 'files', 'a_pdf.pdf') + @thesis.files.detach + + # Attach file with decomposed unicode (i + combining acute accent, not precomposed í) + decomposed_filename = "sald\u0069\u0301as_thesis.pdf" # i (U+0069) + combining acute (U+0301) + @thesis.files.attach(io: File.open(f), filename: decomposed_filename) + @thesis.files.last.purpose = 'thesis_pdf' + @thesis.files.last.description = 'My thesis' + @thesis.save + @thesis.reload + + files = SqsMessage.new(@thesis).map_files + + # Filename should be normalized (decomposed í converted to precomposed) + assert_equal decomposed_filename.unicode_normalize(:nfc), files.first['BitstreamName'] + end end