Fix encoding declarations

CHANGELOG.rst

@@ -9,6 +9,8 @@ latest
   (https://github.com/python-grimp/grimp/issues/317).
 * Fix imports in the ``else`` (or non-type checking ``elif``) branch of an ``if TYPE_CHECKING:``
   statement being excluded by ``exclude_type_checking_imports``, even though they run at runtime.
+* Fix panic when a module declares its encoding using a name that Python accepts but that isn't a
+  WHATWG label, such as ``latin-1`` or ``utf_8``.
 
 3.17 (2026-09-04)
 -----------------

rust/src/errors.rs

@@ -1,6 +1,6 @@
 use crate::exceptions;
 use pyo3::PyErr;
-use pyo3::exceptions::PyValueError;
+use pyo3::exceptions::{PyFileNotFoundError, PyUnicodeError, PyValueError};
 use ruff_python_parser::ParseError as RuffParseError;
 use thiserror::Error;
 
@@ -39,6 +39,12 @@ pub enum GrimpError {
 
     #[error("Cache file {0} was written by a different version of Grimp.")]
     CacheVersionMismatch(String),
+
+    #[error("{0}")]
+    FileNotFound(String),
+
+    #[error("{0}")]
+    UndecodableFile(String),
 }
 
 pub type GrimpResult<T> = Result<T, GrimpError>;
@@ -61,6 +67,9 @@ impl From<GrimpError> for PyErr {
             GrimpError::CacheVersionMismatch(_) => {
                 exceptions::CacheVersionMismatch::new_err(value.to_string())
             }
+            GrimpError::FileNotFound(_) => PyFileNotFoundError::new_err(value.to_string()),
+            // Not UnicodeDecodeError, as that can't be created from just a message.
+            GrimpError::UndecodableFile(_) => PyUnicodeError::new_err(value.to_string()),
         }
     }
 }

rust/src/filesystem.rs

@@ -1,5 +1,6 @@
+use crate::errors::{GrimpError, GrimpResult};
 use itertools::Itertools;
-use pyo3::exceptions::{PyFileNotFoundError, PyTypeError, PyUnicodeDecodeError};
+use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
 use regex::Regex;
 use std::collections::HashMap;
@@ -14,6 +15,48 @@ use unindent::unindent;
 static ENCODING_RE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)").unwrap());
 
+/// An encoding named in a Python source file's encoding declaration.
+enum Encoding {
+    /// True Latin-1 (ISO-8859-1): a straight byte-to-code-point mapping. This is handled
+    /// specially because encoding_rs follows the WHATWG standard, under which the `iso-8859-1`
+    /// label resolves to the Windows-1252 decoder instead (the two differ in the 0x80..=0x9f
+    /// range), and because Windows-1252 has undefined bytes that true Latin-1 decodes fine.
+    Latin1,
+    /// Any other encoding, delegated to encoding_rs.
+    Other(&'static encoding_rs::Encoding),
+}
+
+/// Look up the encoding named in a Python source file's encoding declaration.
+///
+/// encoding_rs only knows the WHATWG labels, but Python accepts other spellings too, such as
+/// `latin-1`, `utf_8`, `utf-8-sig` or `euc_jp`.
+fn lookup_encoding(name: &str) -> Option<Encoding> {
+    // Normalize the name like Python does: see `_get_normal_name` in Lib/tokenize.py, which
+    // special cases UTF-8 and Latin-1. Python's codec lookup also ignores the difference
+    // between underscores and hyphens.
+    //
+    // We check these special cases before deferring to encoding_rs, because some of the Latin-1
+    // aliases (e.g. `iso-8859-1`) are WHATWG labels that encoding_rs would otherwise resolve to
+    // the Windows-1252 decoder.
+    let normalized = name.to_ascii_lowercase().replace('_', "-");
+    let is_spelling_of = |canonical: &str| {
+        normalized == canonical || normalized.starts_with(&format!("{canonical}-"))
+    };
+    if ["latin-1", "iso-8859-1", "iso-latin-1"]
+        .into_iter()
+        .any(is_spelling_of)
+    {
+        return Some(Encoding::Latin1);
+    }
+    if is_spelling_of("utf-8") {
+        return encoding_rs::Encoding::for_label(b"utf-8").map(Encoding::Other);
+    }
+    // Try the name as given, then its normalized form.
+    encoding_rs::Encoding::for_label(name.as_bytes())
+        .or_else(|| encoding_rs::Encoding::for_label(normalized.as_bytes()))
+        .map(Encoding::Other)
+}
+
 pub trait FileSystem: Send + Sync {
     fn sep(&self) -> &str;
 
@@ -23,7 +66,7 @@ pub trait FileSystem: Send + Sync {
 
     fn exists(&self, file_name: &str) -> bool;
 
-    fn read(&self, file_name: &str) -> PyResult<String>;
+    fn read(&self, file_name: &str) -> GrimpResult<String>;
 
     fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()>;
 }
@@ -83,7 +126,7 @@ impl FileSystem for RealBasicFileSystem {
         Path::new(file_name).is_file()
     }
 
-    fn read(&self, file_name: &str) -> PyResult<String> {
+    fn read(&self, file_name: &str) -> GrimpResult<String> {
         // Python files are assumed UTF-8 by default (PEP 686), but they can specify an alternative
         // encoding, which we need to take into account here.
         // See https://peps.python.org/pep-0263/
@@ -92,7 +135,7 @@ impl FileSystem for RealBasicFileSystem {
 
         let path = Path::new(file_name);
         let bytes = fs::read(path).map_err(|e| {
-            PyFileNotFoundError::new_err(format!("Failed to read file {file_name}: {e}"))
+            GrimpError::FileNotFound(format!("Failed to read file {file_name}: {e}"))
         })?;
 
         let s = String::from_utf8_lossy(&bytes);
@@ -110,24 +153,30 @@ impl FileSystem for RealBasicFileSystem {
         }
 
         if let Some(enc_name) = detected_encoding {
-            let encoding =
-                encoding_rs::Encoding::for_label(enc_name.as_bytes()).ok_or_else(|| {
-                    PyUnicodeDecodeError::new_err(format!(
-                        "Failed to decode file {file_name} (unknown encoding '{enc_name}')"
-                    ))
-                })?;
-            let (decoded_s, _, had_errors) = encoding.decode(&bytes);
-            if had_errors {
-                Err(PyUnicodeDecodeError::new_err(format!(
-                    "Failed to decode file {file_name} with encoding '{enc_name}'"
-                )))
-            } else {
-                Ok(decoded_s.into_owned())
+            let encoding = lookup_encoding(&enc_name).ok_or_else(|| {
+                GrimpError::UndecodableFile(format!(
+                    "Failed to decode file {file_name} (unknown encoding '{enc_name}')"
+                ))
+            })?;
+            match encoding {
+                // True Latin-1 maps every byte to the code point of the same value and so can
+                // never fail.
+                Encoding::Latin1 => Ok(bytes.iter().map(|&b| b as char).collect()),
+                Encoding::Other(encoding) => {
+                    let (decoded_s, _, had_errors) = encoding.decode(&bytes);
+                    if had_errors {
+                        Err(GrimpError::UndecodableFile(format!(
+                            "Failed to decode file {file_name} with encoding '{enc_name}'"
+                        )))
+                    } else {
+                        Ok(decoded_s.into_owned())
+                    }
+                }
             }
         } else {
             // Default to UTF-8 if no encoding is specified
             String::from_utf8(bytes).map_err(|e| {
-                PyUnicodeDecodeError::new_err(format!(
+                GrimpError::UndecodableFile(format!(
                     "Failed to decode file {file_name} as UTF-8: {e}"
                 ))
             })
@@ -173,7 +222,7 @@ impl PyRealBasicFileSystem {
     }
 
     fn read(&self, file_name: &str) -> PyResult<String> {
-        self.inner.read(file_name)
+        Ok(self.inner.read(file_name)?)
     }
 
     fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> {
@@ -253,11 +302,11 @@ impl FileSystem for FakeBasicFileSystem {
         self.contents.lock().unwrap().contains_key(file_name)
     }
 
-    fn read(&self, file_name: &str) -> PyResult<String> {
+    fn read(&self, file_name: &str) -> GrimpResult<String> {
         let contents = self.contents.lock().unwrap();
         match contents.get(file_name) {
             Some(file_contents) => Ok(file_contents.clone()),
-            None => Err(PyFileNotFoundError::new_err(format!(
+            None => Err(GrimpError::FileNotFound(format!(
                 "No such file: {file_name}"
             ))),
         }
@@ -301,7 +350,7 @@ impl PyFakeBasicFileSystem {
     }
 
     fn read(&self, file_name: &str) -> PyResult<String> {
-        self.inner.read(file_name)
+        Ok(self.inner.read(file_name)?)
     }
 
     fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> {

rust/src/import_scanning.rs

@@ -130,7 +130,7 @@ fn scan_for_imports_no_py_single_module(
     let found_package_for_module = found_packages_by_module[module];
     let module_filename =
         _determine_module_filename(module, found_package_for_module, file_system).unwrap();
-    let module_contents = file_system.read(&module_filename).unwrap();
+    let module_contents = file_system.read(&module_filename)?;
     let imported_objects =
         import_parsing::parse_imports_from_code(&module_contents, &module_filename)?;
 

tests/functional/test_encoding_handling.py

@@ -1,3 +1,7 @@
+import re
+
+import pytest
+
 import grimp
 
 
@@ -41,3 +45,121 @@ def test_build_graph_of_non_utf8_source():
             "line_contents": "from .imported import π",
         },
     ] == result
+
+
+@pytest.mark.parametrize(
+    "declared_encoding, codec, imported_name",
+    (
+        # Python special cases these spellings of UTF-8 and Latin-1.
+        ("utf_8", "utf-8", "π"),
+        ("UTF-8-sig", "utf-8", "π"),
+        ("latin-1", "latin-1", "jalapeño"),
+        ("latin_1", "latin-1", "jalapeño"),
+        ("iso_8859_1", "latin-1", "jalapeño"),
+        # Python also accepts underscores in place of hyphens in other encoding names.
+        ("euc_jp", "euc_jp", "ラーメン"),
+        ("iso8859_15", "iso8859_15", "jalapeño"),
+    ),
+)
+def test_build_graph_of_source_declaring_encoding_with_python_specific_name(
+    tmp_path, monkeypatch, declared_encoding, codec, imported_name
+):
+    """
+    Tests we can cope with source files that declare their encoding using a name that Python
+    accepts, but that isn't a WHATWG encoding label.
+    """
+    package_directory = tmp_path / "declaredencodingpackage"
+    package_directory.mkdir()
+    (package_directory / "__init__.py").write_text("")
+    (package_directory / "imported.py").write_text("")
+    (package_directory / "importer.py").write_bytes(
+        f"# -*- coding: {declared_encoding} -*-\nfrom .imported import {imported_name}\n".encode(
+            codec
+        )
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    graph = grimp.build_graph("declaredencodingpackage", cache_dir=None)
+
+    result = graph.get_import_details(
+        importer="declaredencodingpackage.importer", imported="declaredencodingpackage.imported"
+    )
+
+    assert [
+        {
+            "importer": "declaredencodingpackage.importer",
+            "imported": "declaredencodingpackage.imported",
+            "is_lazy": False,
+            "line_number": 2,
+            "line_contents": f"from .imported import {imported_name}",
+        },
+    ] == result
+
+
+@pytest.mark.parametrize(
+    "declared_encoding",
+    ("latin-1", "latin_1", "iso-8859-1", "iso_8859_1"),
+)
+def test_build_graph_of_true_latin_1_source(tmp_path, monkeypatch, declared_encoding):
+    """
+    Tests that Latin-1 is decoded as true byte-to-code-point Latin-1, rather than Windows-1252.
+
+    These two encodings only diverge in the 0x80..=0x9f byte range, so we exercise a byte in
+    that range (0x80, which is U+0080 in Latin-1 but U+20AC '€' in Windows-1252).
+    """
+    package_directory = tmp_path / "latin1package"
+    package_directory.mkdir()
+    (package_directory / "__init__.py").write_text("")
+    (package_directory / "imported.py").write_text("")
+    source = f"# -*- coding: {declared_encoding} -*-\nfrom .imported import something  # \x80\n"
+    (package_directory / "importer.py").write_bytes(source.encode("latin-1"))
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    graph = grimp.build_graph("latin1package", cache_dir=None)
+
+    result = graph.get_import_details(
+        importer="latin1package.importer", imported="latin1package.imported"
+    )
+
+    assert [
+        {
+            "importer": "latin1package.importer",
+            "imported": "latin1package.imported",
+            "is_lazy": False,
+            "line_number": 2,
+            "line_contents": "from .imported import something  # \x80",
+        },
+    ] == result
+
+
+@pytest.mark.parametrize(
+    "contents, expected_problem",
+    (
+        pytest.param(b"x = '\xff'\n", "as UTF-8", id="invalid-utf-8"),
+        pytest.param(
+            b"# -*- coding: euc-jp -*-\nx = '\xff\xff'\n",
+            "with encoding 'euc-jp'",
+            id="invalid-for-declared-encoding",
+        ),
+        pytest.param(
+            b"# -*- coding: nonexistent -*-\n",
+            "(unknown encoding 'nonexistent')",
+            id="unknown-encoding",
+        ),
+    ),
+)
+def test_undecodable_source_raises_unicode_error_including_filename(
+    tmp_path, monkeypatch, contents, expected_problem
+):
+    package_directory = tmp_path / "undecodablepackage"
+    package_directory.mkdir()
+    (package_directory / "__init__.py").write_text("")
+    module_filename = package_directory / "undecodable.py"
+    module_filename.write_bytes(contents)
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    with pytest.raises(
+        UnicodeError,
+        match=re.escape(f"Failed to decode file {module_filename} {expected_problem}"),
+    ):
+        grimp.build_graph("undecodablepackage", cache_dir=None)