From 65db1be5e5bbde3fc61f5d40c0b49dc582e363c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=B6rpel?= Date: Wed, 30 Sep 2026 17:54:56 +0200 Subject: [PATCH 1/4] feat: add Tika 4 distribution and response helpers --- tests/test_parser.py | 50 +++++ tests/test_server_install.py | 377 +++++++++++++++++++++++++++++++++++ tests/test_unpack.py | 50 +++++ tika/parser.py | 13 +- tika/tika.py | 298 +++++++++++++++++++++++++-- tika/unpack.py | 58 ++++++ 6 files changed, 821 insertions(+), 25 deletions(-) create mode 100644 tests/test_server_install.py diff --git a/tests/test_parser.py b/tests/test_parser.py index 72d9db81..f1a8cc9c 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,10 +1,60 @@ # SPDX-License-Identifier: Apache-2.0 +import json from http import HTTPStatus from tika import parser +# --- response parsing, no server needed ------------------------------------------- + + +def test_parse_concatenates_content_and_merges_metadata(): + body = json.dumps( + [ + {parser._CONTENT_KEY: "outer ", "Content-Type": "application/pdf", "dc:title": "t"}, + {parser._CONTENT_KEY: "inner", "Content-Type": "image/png"}, + ] + ) + parsed = parser._parse((200, body)) + assert parsed["status"] == 200 + assert parsed["content"] == "outer inner" + assert parsed["metadata"]["Content-Type"] == ["application/pdf", "image/png"] + assert parsed["metadata"]["dc:title"] == "t" + assert parser._CONTENT_KEY not in parsed["metadata"] + + +def test_parse_without_content_yields_none(): + parsed = parser._parse((200, json.dumps([{"Content-Type": "image/png"}]))) + assert parsed["content"] is None + assert parsed["metadata"] == {"Content-Type": "image/png"} + + +def test_parse_meta_service(): + parsed = parser._parse((200, json.dumps({"Content-Type": "application/pdf"})), service="meta") + assert parsed == {"metadata": {"Content-Type": "application/pdf"}, "content": None, "status": 200} + + +def test_parse_text_service_returns_body_verbatim(): + parsed = parser._parse((200, "plain text"), service="text") + assert parsed == {"metadata": None, "content": "plain text", "status": 200} + + +def test_parse_empty_body(): + assert parser._parse((204, "")) == {"metadata": None, "content": None, "status": 204} + assert parser._parse(None) == {"metadata": None, "content": None} + + +def test_parse_does_not_decode_error_bodies(): + parsed = parser._parse((503, json.dumps({"status": "TIMEOUT", "message": "worker timed out"}))) + assert parsed == {"metadata": None, "content": None, "status": 503} + parsed = parser._parse((422, "Unprocessable Entity"), service="meta") + assert parsed == {"metadata": None, "content": None, "status": 422} + + +# --- integration, against the live server ----------------------------------------- + + def test_remote_pdf(remote_fixture_base_url): """parse remote PDF""" assert parser.from_file(f"{remote_fixture_base_url}/remote.pdf") diff --git a/tests/test_server_install.py b/tests/test_server_install.py new file mode 100644 index 00000000..f4b7d42c --- /dev/null +++ b/tests/test_server_install.py @@ -0,0 +1,377 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for locating, downloading, verifying and extracting the Tika server. + +These run without a Tika server, Java or network access. +""" + +import hashlib +import os +import subprocess +import zipfile +from pathlib import Path +from unittest import mock + +import pytest +import requests + +import tika.tika +from tika.tika import TikaDownloadError, TikaException + +DIST_URL = "https://dlcdn.apache.org/tika/9.9.9/tika-server-standard-9.9.9.zip" +ARCHIVE_URL = "https://archive.apache.org/dist/tika/9.9.9/tika-server-standard-9.9.9.zip" +SHA512 = hashlib.sha512(b"x").hexdigest() + + +def make_dist_zip(path, version="9.9.9", top_level_dir=None): + prefix = f"{top_level_dir}/" if top_level_dir else "" + Path(path).parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(f"{prefix}tika-server-standard-{version}.jar", b"launcher") + archive.writestr(f"{prefix}lib/tika-core-{version}.jar", b"core") + archive.writestr(f"{prefix}plugins/tika-pipes-file-system/plugin.jar", b"plugin") + archive.writestr(f"{prefix}LICENSE", b"Apache-2.0") + return path + + +def sha512_of(path): + return hashlib.sha512(Path(path).read_bytes()).hexdigest() + + +@pytest.fixture +def install_env(tmp_path, monkeypatch): + """Point the module-level cache config at a scratch directory.""" + monkeypatch.setattr(tika.tika, "TikaJarPath", str(tmp_path / "cache")) + monkeypatch.setattr(tika.tika, "TikaJarHashAlgo", "sha512") + return tmp_path + + +class FakeDownloader: + """Stand-in for ``_urlretrieve`` serving an in-memory set of files by URL.""" + + def __init__(self, files=None, missing=()): + self.files = dict(files or {}) + self.missing = set(missing) + self.requested = [] + + def __call__(self, url, filename, **kwargs): + self.requested.append(url) + if url in self.missing or url not in self.files: + raise TikaDownloadError(f"Failed to download {url}: 404", 404) + Path(filename).parent.mkdir(parents=True, exist_ok=True) + Path(filename).write_bytes(self.files[url]) + return filename + + +# --- pure helpers ----------------------------------------------------------------- + + +@pytest.mark.parametrize( + "text", + [ + SHA512, + SHA512.upper(), + SHA512 + "\n", + f"{SHA512} tika-server-standard-9.9.9.zip\n", + "tika-server-standard-9.9.9.zip: " + + " ".join(SHA512[i : i + 4].upper() for i in range(0, 64, 4)) + + "\n " + + " ".join(SHA512[i : i + 4].upper() for i in range(64, 128, 4)) + + "\n", + ], +) +def test_parse_checksum_file_accepts_common_layouts(text): + assert tika.tika._parseChecksumFile(text, "sha512") == SHA512 + + +@pytest.mark.parametrize("text", ["", "not a digest", SHA512[:-2], f"{SHA512}ab"]) +def test_parse_checksum_file_rejects_garbage(text): + with pytest.raises(TikaException): + tika.tika._parseChecksumFile(text, "sha512") + + +def test_parse_checksum_file_honours_algorithm(): + md5 = hashlib.md5(b"x").hexdigest() + assert tika.tika._parseChecksumFile(md5, "md5") == md5 + with pytest.raises(TikaException): + tika.tika._parseChecksumFile(md5, "sha512") + + +@pytest.mark.parametrize( + "url, expected", + [ + (DIST_URL, ARCHIVE_URL), + ("https://downloads.apache.org/tika/9.9.9/tika-server-standard-9.9.9.zip", ARCHIVE_URL), + (ARCHIVE_URL, None), + ("https://repo1.maven.org/maven2/org/apache/tika/x.jar", None), + ("https://dlcdn.apache.org/solr/9.0.0/solr.tgz", None), + ("/tmp/tika-server-standard-9.9.9.zip", None), + ], +) +def test_fallback_distribution_url(url, expected): + assert tika.tika._fallbackDistributionUrl(url) == expected + + +def test_local_path_from_url_or_path(tmp_path): + target = tmp_path / "tika-server-standard-9.9.9.zip" + assert tika.tika._localPathFromUrlOrPath(str(target)) == str(target) + assert tika.tika._localPathFromUrlOrPath(target.as_uri()) == str(target) + relative = tika.tika._localPathFromUrlOrPath("dist/x.zip") + assert os.path.isabs(relative) and relative.endswith("x.zip") + windows = tika.tika._localPathFromUrlOrPath(r"C:\tika\x.zip") + assert windows is not None and windows.endswith("x.zip") + assert tika.tika._localPathFromUrlOrPath(DIST_URL) is None + assert tika.tika._localPathFromUrlOrPath("http://localhost:9998/x.zip") is None + + +@pytest.mark.parametrize( + "output, expected", + [ + ('openjdk version "17.0.2" 2022-01-18\nOpenJDK Runtime Environment', 17), + ('java version "1.8.0_292"\nJava(TM) SE Runtime Environment', 8), + ('openjdk version "25.0.4.1" 2026-08-18', 25), + ('Picked up JAVA_TOOL_OPTIONS: -Dhttp.proxyHost=x\nopenjdk version "21" 2023-09-19', 21), + ("gibberish", None), + ], +) +def test_java_major_version(output, expected): + completed = subprocess.CompletedProcess(["java", "-version"], 0, stdout="", stderr=output) + with mock.patch("tika.tika.subprocess.run", return_value=completed): + assert tika.tika._javaMajorVersion("java") == expected + + +def test_java_major_version_missing_binary(): + with mock.patch("tika.tika.subprocess.run", side_effect=FileNotFoundError): + assert tika.tika._javaMajorVersion("/nonexistent/java") is None + + +def test_jar_needs_lib_dir(tmp_path): + thin = tmp_path / "thin.jar" + with zipfile.ZipFile(thin, "w") as jar: + jar.writestr( + "META-INF/MANIFEST.MF", + "Manifest-Version: 1.0\r\nMain-Class: org.apache.tika.server.core.TikaServerCli\r\n" + "Class-Path: lib/tika-server-core-9.9.9.jar lib/tika-core-9.9.9.jar lib/very-long-nam\r\n" + " e-that-wraps.jar\r\n", + ) + fat = tmp_path / "fat.jar" + with zipfile.ZipFile(fat, "w") as jar: + jar.writestr("META-INF/MANIFEST.MF", "Manifest-Version: 1.0\r\nMain-Class: x.Main\r\n") + assert tika.tika._jarNeedsLibDir(str(thin)) is True + assert tika.tika._jarNeedsLibDir(str(fat)) is False + assert tika.tika._jarNeedsLibDir(str(tmp_path / "missing.jar")) is False + + +# --- extraction -------------------------------------------------------------------- + + +def test_extract_server_distribution(tmp_path): + archive = make_dist_zip(tmp_path / "dist.zip") + install_dir = tmp_path / "cache" / "tika-server-standard-9.9.9" + + tika.tika._extractServerDistribution(str(archive), str(install_dir)) + + assert (install_dir / "tika-server-standard-9.9.9.jar").read_bytes() == b"launcher" + assert (install_dir / "lib" / "tika-core-9.9.9.jar").is_file() + assert (install_dir / "plugins" / "tika-pipes-file-system" / "plugin.jar").is_file() + assert os.listdir(install_dir.parent) == ["tika-server-standard-9.9.9"], "no temp dirs left behind" + assert tika.tika._findServerJar(str(install_dir)) == str(install_dir / "tika-server-standard-9.9.9.jar") + + +def test_extract_server_distribution_unwraps_single_top_level_dir(tmp_path): + archive = make_dist_zip(tmp_path / "dist.zip", top_level_dir="tika-server-standard-9.9.9") + install_dir = tmp_path / "tika-server-standard-9.9.9" + tika.tika._extractServerDistribution(str(archive), str(install_dir)) + assert tika.tika._findServerJar(str(install_dir)) == str(install_dir / "tika-server-standard-9.9.9.jar") + + +def test_extract_server_distribution_keeps_existing_install(tmp_path): + archive = make_dist_zip(tmp_path / "dist.zip") + install_dir = tmp_path / "tika-server-standard-9.9.9" + tika.tika._extractServerDistribution(str(archive), str(install_dir)) + (install_dir / "marker").write_text("first") + tika.tika._extractServerDistribution(str(archive), str(install_dir)) + assert (install_dir / "marker").read_text() == "first" + assert sorted(os.listdir(tmp_path)) == ["dist.zip", "tika-server-standard-9.9.9"], "no temp dirs left behind" + + +def test_extract_server_distribution_bad_zip_leaves_nothing(tmp_path): + archive = tmp_path / "dist.zip" + archive.write_bytes(b"not a zip") + install_dir = tmp_path / "tika-server-standard-9.9.9" + with pytest.raises(zipfile.BadZipFile): + tika.tika._extractServerDistribution(str(archive), str(install_dir)) + assert not install_dir.exists() + assert os.listdir(tmp_path) == ["dist.zip"] + + +def test_find_server_jar_requires_lib_dir(tmp_path): + (tmp_path / "tika-server-standard-9.9.9.jar").write_bytes(b"x") + assert tika.tika._findServerJar(str(tmp_path)) is None + (tmp_path / "lib").mkdir() + assert tika.tika._findServerJar(str(tmp_path)) == str(tmp_path / "tika-server-standard-9.9.9.jar") + assert tika.tika._findServerJar(str(tmp_path / "missing")) is None + + +def test_find_server_jar_accepts_single_unnamed_jar(tmp_path): + (tmp_path / "lib").mkdir() + (tmp_path / "custom-build.jar").write_bytes(b"x") + assert tika.tika._findServerJar(str(tmp_path)) == str(tmp_path / "custom-build.jar") + (tmp_path / "another.jar").write_bytes(b"x") + assert tika.tika._findServerJar(str(tmp_path)) is None + + +# --- download layer ---------------------------------------------------------------- + + +def test_urlretrieve_reports_http_status(tmp_path): + response = mock.Mock() + response.raise_for_status.side_effect = requests.HTTPError(response=mock.Mock(status_code=404)) + target = tmp_path / "x.zip" + with mock.patch("tika.tika.requests.get", return_value=response): + with pytest.raises(TikaDownloadError) as excinfo: + tika.tika._urlretrieve("https://example.org/x.zip", str(target)) + assert excinfo.value.status_code == 404 + assert not target.exists() + + +def test_urlretrieve_connection_error_has_no_status(tmp_path): + with mock.patch("tika.tika.requests.get", side_effect=requests.ConnectionError("refused")): + with pytest.raises(TikaDownloadError) as excinfo: + tika.tika._urlretrieve("https://example.org/x.zip", str(tmp_path / "x.zip")) + assert excinfo.value.status_code is None + + +def test_download_with_fallback_retries_on_archive(tmp_path, monkeypatch): + downloader = FakeDownloader({ARCHIVE_URL: b"zip"}, missing={DIST_URL}) + monkeypatch.setattr(tika.tika, "_urlretrieve", downloader) + target = tmp_path / "x.zip" + tika.tika._downloadWithFallback(DIST_URL, str(target)) + assert downloader.requested == [DIST_URL, ARCHIVE_URL] + assert target.read_bytes() == b"zip" + + +def test_download_with_fallback_does_not_retry_other_urls(tmp_path, monkeypatch): + url = "https://repo1.maven.org/maven2/org/apache/tika/x.jar" + downloader = FakeDownloader(missing={url}) + monkeypatch.setattr(tika.tika, "_urlretrieve", downloader) + with pytest.raises(TikaDownloadError): + tika.tika._downloadWithFallback(url, str(tmp_path / "x.jar")) + assert downloader.requested == [url] + + +# --- ensureTikaServerJar ----------------------------------------------------------- + + +@pytest.fixture +def remote_dist(install_env, tmp_path, monkeypatch): + """A fake remote hosting a valid zip and its sha512 sidecar under DIST_URL.""" + archive = make_dist_zip(tmp_path / "upstream.zip") + downloader = FakeDownloader( + {DIST_URL: archive.read_bytes(), DIST_URL + ".sha512": (sha512_of(archive) + "\n").encode()} + ) + monkeypatch.setattr(tika.tika, "_urlretrieve", downloader) + return downloader + + +def test_ensure_downloads_verifies_and_extracts(install_env, remote_dist): + cache = install_env / "cache" + + jar = tika.tika.ensureTikaServerJar(DIST_URL) + + assert jar == str(cache / "tika-server-standard-9.9.9" / "tika-server-standard-9.9.9.jar") + assert (cache / "tika-server-standard-9.9.9" / "lib").is_dir() + assert (cache / "tika-server-standard-9.9.9.zip").is_file(), "archive kept for later re-extraction" + assert (cache / "tika-server-standard-9.9.9.zip.sha512").is_file() + assert remote_dist.requested == [DIST_URL, DIST_URL + ".sha512"] + + +def test_ensure_reuses_extracted_distribution(install_env, remote_dist): + first = tika.tika.ensureTikaServerJar(DIST_URL) + remote_dist.requested.clear() + with mock.patch("tika.tika._fileDigest", side_effect=AssertionError("must not hash again")): + assert tika.tika.ensureTikaServerJar(DIST_URL) == first + assert remote_dist.requested == [] + + +def test_ensure_redownloads_once_on_checksum_mismatch(install_env, remote_dist): + remote_dist.files[DIST_URL + ".sha512"] = (SHA512 + "\n").encode() # wrong digest + with pytest.raises(TikaException, match="Checksum mismatch"): + tika.tika.ensureTikaServerJar(DIST_URL) + assert remote_dist.requested == [DIST_URL, DIST_URL + ".sha512", DIST_URL, DIST_URL + ".sha512"] + assert not (install_env / "cache" / "tika-server-standard-9.9.9").exists() + + +def test_ensure_falls_back_to_archive(install_env, remote_dist): + remote_dist.files[ARCHIVE_URL] = remote_dist.files.pop(DIST_URL) + remote_dist.files[ARCHIVE_URL + ".sha512"] = remote_dist.files.pop(DIST_URL + ".sha512") + + jar = tika.tika.ensureTikaServerJar(DIST_URL) + + assert jar.endswith("tika-server-standard-9.9.9.jar") + assert remote_dist.requested == [DIST_URL, ARCHIVE_URL, DIST_URL + ".sha512", ARCHIVE_URL + ".sha512"] + + +def test_ensure_local_zip_via_file_url(install_env, tmp_path, monkeypatch): + archive = make_dist_zip(tmp_path / "offline" / "tika-server-standard-9.9.9.zip") + (tmp_path / "offline" / "tika-server-standard-9.9.9.zip.sha512").write_text(sha512_of(archive)) + monkeypatch.setattr(tika.tika, "_urlretrieve", mock.Mock(side_effect=AssertionError("no network"))) + + jar = tika.tika.ensureTikaServerJar(archive.as_uri()) + + assert jar == str(install_env / "cache" / "tika-server-standard-9.9.9" / "tika-server-standard-9.9.9.jar") + assert archive.is_file(), "the user's archive is left in place" + assert not (install_env / "cache" / "tika-server-standard-9.9.9.zip").exists(), "not copied" + + +def test_ensure_local_zip_plain_path_without_sidecar_warns(install_env, tmp_path, monkeypatch, caplog): + archive = make_dist_zip(tmp_path / "offline" / "tika-server-standard-9.9.9.zip") + monkeypatch.setattr(tika.tika, "_urlretrieve", mock.Mock(side_effect=AssertionError("no network"))) + + with caplog.at_level("WARNING", logger="tika.tika"): + jar = tika.tika.ensureTikaServerJar(str(archive)) + + assert jar.endswith("tika-server-standard-9.9.9.jar") + assert "skipping verification" in caplog.text + + +def test_ensure_local_zip_with_bad_sidecar_fails_without_deleting(install_env, tmp_path): + archive = make_dist_zip(tmp_path / "offline" / "tika-server-standard-9.9.9.zip") + (tmp_path / "offline" / "tika-server-standard-9.9.9.zip.sha512").write_text(SHA512) + + with pytest.raises(TikaException, match="Checksum mismatch"): + tika.tika.ensureTikaServerJar(str(archive)) + + assert archive.is_file() + assert not (install_env / "cache" / "tika-server-standard-9.9.9").exists() + + +def test_ensure_local_jar_override_is_used_as_is(install_env, tmp_path): + unpacked = tmp_path / "unpacked" + (unpacked / "lib").mkdir(parents=True) + jar = unpacked / "tika-server-standard-9.9.9.jar" + jar.write_bytes(b"launcher") + + assert tika.tika.ensureTikaServerJar(str(jar)) == str(jar) + assert tika.tika.ensureTikaServerJar(jar.as_uri()) == str(jar) + with pytest.raises(TikaException, match="does not exist"): + tika.tika.ensureTikaServerJar(str(tmp_path / "missing.jar")) + + +def test_ensure_remote_jar_override_is_downloaded_and_verified(install_env, monkeypatch): + url = "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.3.2/tika-server-standard-3.3.2.jar" + downloader = FakeDownloader({url: b"fat jar", url + ".sha512": hashlib.sha512(b"fat jar").hexdigest().encode()}) + monkeypatch.setattr(tika.tika, "_urlretrieve", downloader) + + jar = tika.tika.ensureTikaServerJar(url) + + assert jar == str(install_env / "cache" / "tika-server-standard-3.3.2.jar") + assert Path(jar).read_bytes() == b"fat jar" + assert downloader.requested == [url, url + ".sha512"] + downloader.requested.clear() + assert tika.tika.ensureTikaServerJar(url) == jar + assert downloader.requested == [], "cached jar is reused" + + +def test_ensure_rejects_url_without_file_name(install_env): + with pytest.raises(TikaException, match="file name"): + tika.tika.ensureTikaServerJar("https://dlcdn.apache.org/tika/") diff --git a/tests/test_unpack.py b/tests/test_unpack.py index 04a6fd43..d61cfc27 100644 --- a/tests/test_unpack.py +++ b/tests/test_unpack.py @@ -1,5 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 +import io +import json +import zipfile + from tika import unpack # Test data @@ -7,6 +11,52 @@ TEXT_ASCII = "Hello, world!!" +# --- zip response parsing, no server needed --------------------------------------- + + +def make_unpack_zip(entries): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name, data in entries.items(): + archive.writestr(name, json.dumps(data) if isinstance(data, dict) else data) + return buffer.getvalue() + + +def test_parse_zip_splits_container_and_attachments(): + body = make_unpack_zip( + { + "0.pdf": b"%PDF-original", + "0.pdf.metadata.json": {"tk:content": "hello", "Content-Type": "application/pdf"}, + "1.png": b"\x89PNG", + "1.png.metadata.json": {"Content-Type": "image/png"}, + "2.docx": b"PK", + } + ) + parsed = unpack._parse_zip((200, body)) + assert parsed["content"] == "hello" + assert parsed["metadata"] == {"Content-Type": "application/pdf"} + assert parsed["attachments"] == {"1.png": b"\x89PNG", "2.docx": b"PK"} + + +def test_parse_zip_accepts_stem_named_sidecar(): + body = make_unpack_zip({"0.txt": b"hi", "0.metadata.json": {"tk:content": "hi", "Content-Type": "text/plain"}}) + parsed = unpack._parse_zip((200, body)) + assert parsed["content"] == "hi" + assert parsed["metadata"]["Content-Type"] == "text/plain" + assert parsed["attachments"] == {} + + +def test_parse_zip_without_sidecar(): + parsed = unpack._parse_zip((200, make_unpack_zip({"0.txt": b"hi"}))) + assert parsed == {"content": "", "metadata": {}, "attachments": {}} + + +def test_parse_zip_empty_response(): + assert unpack._parse_zip((204, b"")) == {} + assert unpack._parse_zip((200, None)) == {} + assert unpack._parse_zip(None) == {} + + def test_utf8(tmp_path): """Test UTF-8 encoding""" test_file = tmp_path / "test_utf8.txt" diff --git a/tika/parser.py b/tika/parser.py index 118477bd..c6e8b345 100644 --- a/tika/parser.py +++ b/tika/parser.py @@ -18,6 +18,9 @@ from .tika import ServerEndpoint, callServer, parse1 +# metadata key under which the /rmeta endpoint returns the extracted text of each document +_CONTENT_KEY = 'X-TIKA:content' + def from_file(filename, serverEndpoint=ServerEndpoint, service='all', xmlContent=False, headers=None, config_path=None, requestOptions={}, raw_response=False): ''' @@ -92,6 +95,10 @@ def _parse(output, service='all'): parsed["content"] = output[1] return parsed + if output[0] != 200: + # an error body (JSON since Tika 4, plain text before) is not a parse result + return parsed + realJson = json.loads(output[1]) parsed["metadata"] = {} @@ -102,8 +109,8 @@ def _parse(output, service='all'): content = "" for js in realJson: - if "X-TIKA:content" in js: - content += js["X-TIKA:content"] + if _CONTENT_KEY in js: + content += js[_CONTENT_KEY] if content == "": content = None @@ -112,7 +119,7 @@ def _parse(output, service='all'): for js in realJson: for n in js: - if n != "X-TIKA:content": + if n != _CONTENT_KEY: if n in parsed["metadata"]: if not isinstance(parsed["metadata"][n], list): parsed["metadata"][n] = [parsed["metadata"][n]] diff --git a/tika/tika.py b/tika/tika.py index fa351f10..6966104d 100755 --- a/tika/tika.py +++ b/tika/tika.py @@ -123,11 +123,15 @@ def make_content_disposition_header(fn): import io import logging import platform +import shutil import signal import socket +import subprocess import tempfile +import zipfile from os import walk from subprocess import STDOUT, Popen +from urllib.request import url2pathname import requests @@ -156,6 +160,11 @@ def make_content_disposition_header(fn): TikaJarPath = os.getenv('TIKA_PATH', tempfile.gettempdir()) TikaFilesPath = tempfile.gettempdir() TikaServerLogFilePath = log_path +# Apache Tika 4 ships the runnable server as a zip distribution in the Apache dist area; +# current releases live on the CDN, every release stays on the archive. +TikaDistBaseUrl = 'https://dlcdn.apache.org/tika/' +TikaArchiveBaseUrl = 'https://archive.apache.org/dist/tika/' +TikaMinJavaVersion = 17 TikaServerJar = os.getenv( 'TIKA_SERVER_JAR', "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/"+TikaVersion+"/tika-server-standard-"+TikaVersion+".jar") @@ -184,6 +193,15 @@ def make_content_disposition_header(fn): class TikaException(Exception): pass +class TikaDownloadError(RuntimeError): + ''' + Raised when a download fails. ``status_code`` holds the HTTP status when the + server answered, ``None`` for connection level failures. + ''' + def __init__(self, message, status_code=None): + super().__init__(message) + self.status_code = status_code + def echo2(*s): sys.stderr.write(str('tika.py: %s\n') % str(' ').join(map(str, s))) def warn(*s): echo2('Warn:', *s) def die(*s): warn('Error:', *s); echo2(USAGE); sys.exit() @@ -591,22 +609,33 @@ def checkTikaServer(scheme="http", serverHost=ServerHost, port=Port, tikaServerJ def checkJarSig(tikaServerJar, jarPath): ''' - Checks the signature of Jar - :param tikaServerJar: - :param jarPath: - :return: ``True`` if the signature of the jar matches + Verifies a downloaded server archive (or jar) against its ``.`` + checksum sidecar. For a remote source the sidecar is downloaded next to ``jarPath`` + when it is missing. For a local source (plain path or ``file://`` URL) the sidecar is + looked up next to the source; when there is none the check is skipped with a warning + so that air-gapped installs keep working. + :param tikaServerJar: URL or local path the archive came from + :param jarPath: local path of the archive to verify + :return: ``True`` if the digest matches (or no sidecar is available for a local source) ''' localChecksumPath = '.'.join([jarPath, TikaJarHashAlgo]) if not os.path.isfile(localChecksumPath): - remoteChecksum = '.'.join([tikaServerJar, TikaJarHashAlgo]) - getRemoteJar(remoteChecksum, localChecksumPath) - m = hashlib.new(TikaJarHashAlgo) - with open(jarPath, 'rb') as f: - binContents = f.read() - m.update(binContents) - with open(f"{jarPath}.{TikaJarHashAlgo}", "r") as em: - existingContents = em.read() - return existingContents == m.hexdigest() + sourcePath = _localPathFromUrlOrPath(tikaServerJar) + if sourcePath is None: + remoteChecksum = '.'.join([tikaServerJar, TikaJarHashAlgo]) + getRemoteJar(remoteChecksum, localChecksumPath) + else: + localChecksumPath = '.'.join([sourcePath, TikaJarHashAlgo]) + if not os.path.isfile(localChecksumPath): + log.warning('No %s checksum found next to %s; skipping verification.', TikaJarHashAlgo, sourcePath) + return True + with open(localChecksumPath, 'r') as em: + try: + expected = _parseChecksumFile(em.read(), TikaJarHashAlgo) + except TikaException as e: + log.warning('%s: %s', localChecksumPath, e) + return False + return expected == _fileDigest(jarPath, TikaJarHashAlgo) def startServer(tikaServerJar, java_path = TikaJava, java_args = TikaJavaArgs, serverHost = ServerHost, port = Port, classpath=None, config_path=None): @@ -746,7 +775,7 @@ def _urlretrieve( The filepath where the file was saved. Raises: - requests.RequestException: If the download fails. + TikaDownloadError: If the download fails (``status_code`` set for HTTP errors). IOError: If there's an issue writing to the file. """ headers = {"user-agent": "tika-python"} @@ -775,7 +804,8 @@ def _urlretrieve( # Clean up partial file on error if os.path.exists(filename): os.remove(filename) - raise RuntimeError(f"Failed to download {url}: {e}") from e + status_code = getattr(getattr(e, 'response', None), 'status_code', None) + raise TikaDownloadError(f"Failed to download {url}: {e}", status_code) from e def getRemoteFile(urlOrPath, destPath): @@ -803,18 +833,242 @@ def getRemoteFile(urlOrPath, destPath): def getRemoteJar(urlOrPath, destPath): ''' - Fetches URL to local path or just return absolute path. - :param urlOrPath: remote resource locator + Fetches URL to local path or just return the local path. + :param urlOrPath: remote resource locator, ``file://`` URL or local path :param destPath: Path to store the resource, usually a path on file system :return: tuple having (path, 'local'/'remote') ''' + localPath = _localPathFromUrlOrPath(urlOrPath) + if localPath is not None: + return (localPath, 'local') + log.info('Retrieving %s to %s.' % (urlOrPath, destPath)) + _downloadWithFallback(urlOrPath, destPath) + return (destPath, 'remote') + + +def _localPathFromUrlOrPath(urlOrPath): + ''' + Returns the local filesystem path for a plain path or a ``file://`` URL, or ``None`` + when ``urlOrPath`` points at a remote resource. + ''' urlp = urlparse(urlOrPath) - if urlp.scheme == '': - return (os.path.abspath(urlOrPath), 'local') + if urlp.scheme == 'file': + return os.path.abspath(url2pathname(urlp.path)) + if len(urlp.scheme) <= 1: + # no scheme, or a single letter which is a Windows drive such as C:\... + return os.path.abspath(urlOrPath) + return None + + +def _fallbackDistributionUrl(url): + ''' + Maps a URL on the Apache dist mirrors (which only carry current releases) to the + permanent copy on archive.apache.org. Returns ``None`` for any other URL. + ''' + urlp = urlparse(url) + if urlp.scheme in ('http', 'https') and urlp.hostname in ('dlcdn.apache.org', 'downloads.apache.org') \ + and urlp.path.startswith('/tika/'): + return TikaArchiveBaseUrl + urlp.path[len('/tika/'):] + return None + + +def _downloadWithFallback(url, filename): + ''' + Downloads ``url`` to ``filename``. When an Apache dist mirror no longer carries the + requested release (404) the download is retried once from archive.apache.org. + ''' + try: + return _urlretrieve(url, filename) + except TikaDownloadError as e: + fallback = _fallbackDistributionUrl(url) + if e.status_code != 404 or fallback is None: + raise + log.warning('%s not found (404); retrying from %s', url, fallback) + return _urlretrieve(fallback, filename) + + +def _parseChecksumFile(text, algo): + ''' + Extracts the hex digest from a checksum sidecar. Accepts a bare digest, the + `` `` form and the ``gpg --print-md`` block form. + :return: lowercase hex digest + ''' + expected = hashlib.new(algo).digest_size * 2 + for token in text.split(): + if len(token) == expected and re.fullmatch(r'[0-9a-fA-F]+', token): + return token.lower() + # gpg --print-md style: ": AB12 CD34 ..." possibly wrapped over several lines + compact = re.sub(r'[^0-9a-fA-F]', '', text.split(':', 1)[-1]) + if len(compact) == expected: + return compact.lower() + raise TikaException('No %s digest found in checksum file' % algo) + + +def _fileDigest(path, algo, chunk_size=1 << 20): + ''' + Hex digest of a file, computed in chunks. + ''' + m = hashlib.new(algo) + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(chunk_size), b''): + m.update(chunk) + return m.hexdigest() + + +def _findServerJar(installDir): + ''' + Returns the path of the Tika server launcher jar inside an extracted distribution, + or ``None`` if ``installDir`` does not hold a complete distribution (jar plus ``lib/``). + ''' + if not os.path.isdir(os.path.join(installDir, 'lib')): + return None + jars = [n for n in os.listdir(installDir) if n.lower().endswith('.jar')] + preferred = sorted(n for n in jars if n.startswith('tika-server-standard-')) + if preferred: + return os.path.join(installDir, preferred[0]) + if len(jars) == 1: + return os.path.join(installDir, jars[0]) + return None + + +def _extractServerDistribution(archivePath, installDir): + ''' + Extracts the Tika server zip distribution into ``installDir``. The archive is unpacked + into a temporary sibling directory that is renamed into place at the end, so + ``installDir`` only ever exists once it is complete. + :return: ``installDir`` + ''' + installDir = os.path.abspath(installDir) + parent = os.path.dirname(installDir) + os.makedirs(parent, exist_ok=True) + tmpDir = tempfile.mkdtemp(prefix=os.path.basename(installDir) + '.', dir=parent) + try: + with zipfile.ZipFile(archivePath) as archive: + archive.extractall(tmpDir) + # tolerate a distribution wrapped in a single top-level directory + source = tmpDir + entries = os.listdir(tmpDir) + if len(entries) == 1 and _findServerJar(os.path.join(tmpDir, entries[0])): + source = os.path.join(tmpDir, entries[0]) + try: + os.rename(source, installDir) + except OSError: + if not _findServerJar(installDir): + raise + # another process extracted the same distribution first; keep theirs + finally: + shutil.rmtree(tmpDir, ignore_errors=True) + return installDir + + +def _javaMajorVersion(java_path=TikaJava): + ''' + Major version of the given Java runtime (8 for 1.8.x, 17 for 17.x), or ``None`` when + the binary cannot be run or its version output cannot be parsed. + ''' + try: + proc = subprocess.run([java_path, '-version'], capture_output=True, text=True, timeout=30) + except (OSError, subprocess.SubprocessError): + return None + match = re.search(r'version "(\d+)(?:\.(\d+))?', proc.stderr + proc.stdout) + if not match: + return None + major = int(match.group(1)) + if major == 1 and match.group(2): + return int(match.group(2)) + return major + + +def _jarNeedsLibDir(jarPath): + ''' + ``True`` when the jar's manifest references a ``lib/`` class path, i.e. it is a thin + launcher that only runs next to the ``lib/`` directory of its distribution. + ''' + try: + with zipfile.ZipFile(jarPath) as jar: + manifest = jar.read('META-INF/MANIFEST.MF').decode('utf-8', 'replace') + except (OSError, KeyError, zipfile.BadZipFile): + return False + manifest = manifest.replace('\r\n ', '').replace('\n ', '') # unwrap continuation lines + return bool(re.search(r'^Class-Path:.*\blib/', manifest, re.MULTILINE)) + + +def _removeQuietly(*paths): + for path in paths: + try: + os.remove(path) + except OSError: + pass + + +def ensureTikaServerJar(tikaServerJar=TikaServerJar): + ''' + Makes sure a runnable Tika server is available locally and returns the path of its + launcher jar. + + ``tikaServerJar`` is normally the URL of the ``tika-server-standard-.zip`` + distribution. It may also be a ``file://`` URL or plain path to a local copy of that + zip, or a URL/path ending in ``.jar`` for a jar that already sits next to its ``lib/`` + directory (or a self-contained pre-4.0 jar). Remote downloads are verified against + their ``.`` sidecar; a zip is extracted once into + ``//`` and reused on later runs. + :return: path of the server jar to launch + ''' + sourcePath = _localPathFromUrlOrPath(tikaServerJar) + name = os.path.basename(sourcePath if sourcePath is not None else urlparse(tikaServerJar).path) + if not name: + raise TikaException('Cannot determine the file name of the Tika server distribution from %r' % (tikaServerJar,)) + checksumSuffix = '.' + TikaJarHashAlgo + + if name.lower().endswith('.jar'): + if sourcePath is not None: + if not os.path.isfile(sourcePath): + raise TikaException('Tika server jar %s does not exist' % sourcePath) + jarPath = sourcePath + else: + jarPath = os.path.join(TikaJarPath, name) + if not os.path.isfile(jarPath): + getRemoteJar(tikaServerJar, jarPath) + if not checkJarSig(tikaServerJar, jarPath): + log.warning('Checksum mismatch for %s; downloading again.', jarPath) + _removeQuietly(jarPath, jarPath + checksumSuffix) + getRemoteJar(tikaServerJar, jarPath) + if not checkJarSig(tikaServerJar, jarPath): + raise TikaException('Checksum mismatch for %s after re-download' % jarPath) + if _jarNeedsLibDir(jarPath) and not os.path.isdir(os.path.join(os.path.dirname(jarPath), 'lib')): + log.warning('%s is a thin launcher without a lib/ directory next to it; since Apache Tika 4 ' + 'the server only runs from inside the unzipped tika-server-standard distribution.', jarPath) + return jarPath + + stem = name[:-4] if name.lower().endswith('.zip') else name + installDir = os.path.join(TikaJarPath, stem) + jarPath = _findServerJar(installDir) + if jarPath: + return jarPath + + if sourcePath is not None: + archivePath = sourcePath + if not os.path.isfile(archivePath): + raise TikaException('Tika server distribution %s does not exist' % archivePath) + if not checkJarSig(tikaServerJar, archivePath): + raise TikaException('Checksum mismatch for %s' % archivePath) else: - log.info('Retrieving %s to %s.' % (urlOrPath, destPath)) - _urlretrieve(urlOrPath, destPath) - return (destPath, 'remote') + archivePath = os.path.join(TikaJarPath, name) + if not os.path.isfile(archivePath): + getRemoteJar(tikaServerJar, archivePath) + if not checkJarSig(tikaServerJar, archivePath): + log.warning('Checksum mismatch for %s; downloading again.', archivePath) + _removeQuietly(archivePath, archivePath + checksumSuffix) + getRemoteJar(tikaServerJar, archivePath) + if not checkJarSig(tikaServerJar, archivePath): + raise TikaException('Checksum mismatch for %s after re-download' % archivePath) + + log.info('Extracting %s to %s.', archivePath, installDir) + _extractServerDistribution(archivePath, installDir) + jarPath = _findServerJar(installDir) + if not jarPath: + raise TikaException('No tika-server-standard-*.jar with a lib/ directory found in %s' % installDir) + return jarPath def checkPortIsOpen(remoteServerHost=ServerHost, port = Port): ''' diff --git a/tika/unpack.py b/tika/unpack.py index 0850e7af..01cb9fb1 100644 --- a/tika/unpack.py +++ b/tika/unpack.py @@ -15,7 +15,10 @@ # import csv +import json +import re import tarfile +import zipfile from contextlib import closing from io import BytesIO, TextIOWrapper @@ -23,6 +26,61 @@ _text_wrapper = TextIOWrapper +# Tika 4 /unpack/all output: a zip holding the original document as ``0.``, embedded +# documents as ``1.``, ``2.``, ... and one ``.metadata.json`` sidecar per entry. +UNPACK_SERVICE = '/unpack/all/text' +_SIDECAR_SUFFIX = '.metadata.json' +_CONTENT_KEY = 'tk:content' + + +def _parse_zip(zipOutput): + ''' + Turns a Tika 4 ``/unpack/all`` zip response into the same shape ``_parse`` produced for + the pre-4.0 tar output: ``content`` and ``metadata`` of the container document (taken + from its ``.metadata.json`` sidecar) and ``attachments`` mapping the name of each + embedded document to its bytes. Sidecars and the container itself are not attachments. + :param zipOutput: ``(status, bytes)`` tuple as returned by ``callServer(rawResponse=True)`` + :return: ``{}`` for an empty response, else a dict with ``content``, ``metadata``, ``attachments`` + ''' + if not zipOutput or not zipOutput[1]: + return {} + + with zipfile.ZipFile(BytesIO(zipOutput[1])) as archive: + names = [info.filename for info in archive.infolist() if not info.is_dir()] + sidecars = {n for n in names if n.endswith(_SIDECAR_SUFFIX)} + files = sorted(n for n in names if n not in sidecars) + + container = next((n for n in files if re.match(r'0(\.|$)', n)), files[0] if files else None) + + metadata = {} + for candidate in _sidecar_candidates(container, sidecars): + metadata = json.loads(archive.read(candidate).decode('utf-8')) + break + if isinstance(metadata, list): # tolerate an rmeta style list; the container comes first + metadata = metadata[0] if metadata else {} + + content = metadata.pop(_CONTENT_KEY, '') + if isinstance(content, list): + content = ''.join(content) + + attachments = {n: archive.read(n) for n in files if n != container} + + return {'content': content, 'metadata': metadata, 'attachments': attachments} + + +def _sidecar_candidates(container, sidecars): + ''' + Sidecar names that may describe the container document, most specific first: + ``0.pdf.metadata.json``, then ``0.metadata.json``. + ''' + candidates = [] + if container is not None: + candidates.append(container + _SIDECAR_SUFFIX) + stem = container.rsplit('.', 1)[0] + candidates.append(stem + _SIDECAR_SUFFIX) + candidates.append('0' + _SIDECAR_SUFFIX) + return [c for c in candidates if c in sidecars] + def from_file(filename, serverEndpoint=ServerEndpoint, requestOptions={}): ''' From 0a43506fd0acc9482ef70bc0c2c9b93890f9f528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=B6rpel?= Date: Wed, 30 Sep 2026 18:06:18 +0200 Subject: [PATCH 2/4] feat: switch to Apache Tika 4.1.0 --- .github/workflows/ci.yml | 7 ++ tests/test_language.py | 7 +- tests/test_unpack.py | 30 ++++++-- tika/__init__.py | 2 +- tika/detector.py | 2 +- tika/language.py | 2 +- tika/parser.py | 5 +- tika/tika.py | 112 ++++++++++++++++++++--------- tika/unpack.py | 149 +++++++++++---------------------------- 9 files changed, 162 insertions(+), 154 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45edad25..13ebdd6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,13 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip + # Apache Tika 4 requires Java 17+; test against the minimum + - name: Set up Java 17 + uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: "17" + - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/tests/test_language.py b/tests/test_language.py index 703aee5f..63f724aa 100644 --- a/tests/test_language.py +++ b/tests/test_language.py @@ -13,4 +13,9 @@ def test_local_path(test_file_path): def test_local_buffer(): - assert language.from_buffer("Good evening, David. How are you?") == "en" + assert ( + language.from_buffer( + "Good evening, David. How are you? The weather has been lovely this week and the garden is finally in bloom." + ) + == "en" + ) diff --git a/tests/test_unpack.py b/tests/test_unpack.py index d61cfc27..06279594 100644 --- a/tests/test_unpack.py +++ b/tests/test_unpack.py @@ -32,7 +32,7 @@ def test_parse_zip_splits_container_and_attachments(): "2.docx": b"PK", } ) - parsed = unpack._parse_zip((200, body)) + parsed = unpack._parse((200, body)) assert parsed["content"] == "hello" assert parsed["metadata"] == {"Content-Type": "application/pdf"} assert parsed["attachments"] == {"1.png": b"\x89PNG", "2.docx": b"PK"} @@ -40,21 +40,21 @@ def test_parse_zip_splits_container_and_attachments(): def test_parse_zip_accepts_stem_named_sidecar(): body = make_unpack_zip({"0.txt": b"hi", "0.metadata.json": {"tk:content": "hi", "Content-Type": "text/plain"}}) - parsed = unpack._parse_zip((200, body)) + parsed = unpack._parse((200, body)) assert parsed["content"] == "hi" assert parsed["metadata"]["Content-Type"] == "text/plain" assert parsed["attachments"] == {} def test_parse_zip_without_sidecar(): - parsed = unpack._parse_zip((200, make_unpack_zip({"0.txt": b"hi"}))) + parsed = unpack._parse((200, make_unpack_zip({"0.txt": b"hi"}))) assert parsed == {"content": "", "metadata": {}, "attachments": {}} def test_parse_zip_empty_response(): - assert unpack._parse_zip((204, b"")) == {} - assert unpack._parse_zip((200, None)) == {} - assert unpack._parse_zip(None) == {} + assert unpack._parse((204, b"")) == {} + assert unpack._parse((200, None)) == {} + assert unpack._parse(None) == {} def test_utf8(tmp_path): @@ -63,6 +63,7 @@ def test_utf8(tmp_path): test_file.write_bytes(TEXT_UTF8.encode("utf8")) parsed = unpack.from_file(str(test_file)) assert parsed["content"].strip() == TEXT_UTF8 + assert parsed["attachments"] == {} def test_ascii(tmp_path): @@ -71,6 +72,21 @@ def test_ascii(tmp_path): test_file.write_text(TEXT_ASCII) parsed = unpack.from_file(str(test_file)) assert parsed["content"].strip() == TEXT_ASCII + assert parsed["metadata"]["Content-Type"].startswith("text/plain") + assert parsed["attachments"] == {} + + +def test_embedded_documents_become_attachments(tmp_path): + """A zip is a container: its members come back as attachments, its own text as content""" + container = tmp_path / "container.zip" + with zipfile.ZipFile(container, "w") as archive: + archive.writestr("a.txt", "alpha text") + archive.writestr("b.txt", "bravo text") + parsed = unpack.from_file(str(container)) + assert parsed["metadata"]["Content-Type"] == "application/zip" + assert "a.txt" in parsed["content"] and "b.txt" in parsed["content"] + assert sorted(parsed["attachments"].values()) == [b"alpha text", b"bravo text"] + assert all(name.startswith(("1.", "2.")) for name in parsed["attachments"]), parsed["attachments"].keys() def test_from_buffer(): @@ -78,6 +94,8 @@ def test_from_buffer(): assert parsed is not None assert parsed["metadata"] is not None assert parsed["metadata"]["Content-Length"] == "5" + assert parsed["content"].strip() == "what?" + assert parsed["attachments"] == {} def test_from_buffer_with_headers(): diff --git a/tika/__init__.py b/tika/__init__.py index 7e0a03b5..079856e0 100644 --- a/tika/__init__.py +++ b/tika/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.3.3.dev0" +__version__ = "4.1.0.dev0" from pkgutil import extend_path diff --git a/tika/detector.py b/tika/detector.py index b561545d..66a22b22 100644 --- a/tika/detector.py +++ b/tika/detector.py @@ -32,6 +32,6 @@ def from_buffer(string, config_path=None, requestOptions={}): :param string: buffered content whose type needs to be detected :return: ''' - status, response = callServer('put', ServerEndpoint, '/detect/stream', string, + status, response = callServer('put', ServerEndpoint, '/detect', string, {'Accept': 'text/plain'}, False, config_path=config_path, requestOptions=requestOptions) return response diff --git a/tika/language.py b/tika/language.py index 7f38c960..039d8d39 100644 --- a/tika/language.py +++ b/tika/language.py @@ -32,6 +32,6 @@ def from_buffer(string, requestOptions={}): :param string: buffered data :return: ''' - status, response = callServer('put', ServerEndpoint, '/language/string', string, + status, response = callServer('put', ServerEndpoint, '/language', string, {'Accept': 'text/plain'}, False, requestOptions=requestOptions) return response diff --git a/tika/parser.py b/tika/parser.py index c6e8b345..e0abf9f3 100644 --- a/tika/parser.py +++ b/tika/parser.py @@ -19,7 +19,8 @@ from .tika import ServerEndpoint, callServer, parse1 # metadata key under which the /rmeta endpoint returns the extracted text of each document -_CONTENT_KEY = 'X-TIKA:content' +# (X-TIKA:content before Tika 4.0) +_CONTENT_KEY = 'tk:content' def from_file(filename, serverEndpoint=ServerEndpoint, service='all', xmlContent=False, headers=None, config_path=None, requestOptions={}, raw_response=False): @@ -41,7 +42,7 @@ def from_file(filename, serverEndpoint=ServerEndpoint, service='all', xmlContent if not xmlContent: output = parse1(service, filename, serverEndpoint, headers=headers, config_path=config_path, requestOptions=requestOptions) else: - output = parse1(service, filename, serverEndpoint, services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/xml'}, + output = parse1(service, filename, serverEndpoint, services={'meta': '/meta', 'text': '/tika/xml', 'all': '/rmeta/xml'}, headers=headers, config_path=config_path, requestOptions=requestOptions) if raw_response: return output diff --git a/tika/tika.py b/tika/tika.py index 6966104d..d4f995f3 100755 --- a/tika/tika.py +++ b/tika/tika.py @@ -156,7 +156,7 @@ def make_content_disposition_header(fn): log.setLevel(logging.INFO) Windows = True if platform.system() == "Windows" else False -TikaVersion = os.getenv('TIKA_VERSION', '3.3.2') +TikaVersion = os.getenv('TIKA_VERSION', '4.1.0') TikaJarPath = os.getenv('TIKA_PATH', tempfile.gettempdir()) TikaFilesPath = tempfile.gettempdir() TikaServerLogFilePath = log_path @@ -164,11 +164,21 @@ def make_content_disposition_header(fn): # current releases live on the CDN, every release stays on the archive. TikaDistBaseUrl = 'https://dlcdn.apache.org/tika/' TikaArchiveBaseUrl = 'https://archive.apache.org/dist/tika/' -TikaMinJavaVersion = 17 TikaServerJar = os.getenv( 'TIKA_SERVER_JAR', - "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/"+TikaVersion+"/tika-server-standard-"+TikaVersion+".jar") -TikaJarHashAlgo=os.getenv('TIKA_JAR_HASH_ALGO', 'md5') + TikaDistBaseUrl + TikaVersion + '/tika-server-standard-' + TikaVersion + '.zip') +TikaJarHashAlgo = os.getenv('TIKA_JAR_HASH_ALGO', 'sha512') + + +def _tikaMajorVersion(version): + try: + return int(str(version).split('.')[0]) + except ValueError: + return None + + +# Tika 4.x needs Java 17; older servers (still usable via TIKA_VERSION/TIKA_SERVER_JAR) ran on 11 +TikaMinJavaVersion = 11 if (_tikaMajorVersion(TikaVersion) or 4) < 4 else 17 ServerHost = "localhost" Port = "9998" ServerEndpoint = os.getenv( @@ -179,7 +189,8 @@ def make_content_disposition_header(fn): TikaClientOnly = os.getenv('TIKA_CLIENT_ONLY', False) TikaServerClasspath = os.getenv('TIKA_SERVER_CLASSPATH', '') TikaStartupSleep = float(os.getenv('TIKA_STARTUP_SLEEP', 5)) -TikaStartupMaxRetry = int(os.getenv('TIKA_STARTUP_MAX_RETRY', 3)) +# Tika 4 boots its parse worker pool before reporting "Started"; allow up to ~30 s on slow machines +TikaStartupMaxRetry = int(os.getenv('TIKA_STARTUP_MAX_RETRY', 6)) TikaJava = os.getenv("TIKA_JAVA", "java") TikaJavaArgs = os.getenv("TIKA_JAVA_ARGS", '') @@ -193,6 +204,16 @@ def make_content_disposition_header(fn): class TikaException(Exception): pass +# Accept header expected by each /tika output path (Tika 4 no longer picks the format from Accept) +_TikaAcceptTypes = { + '/tika': 'text/plain', + '/tika/text': 'text/plain', + '/tika/md': 'text/plain', + '/tika/xml': 'text/xml', + '/tika/html': 'text/html', + '/tika/json': 'application/json', +} + class TikaDownloadError(RuntimeError): ''' Raised when a download fails. ``status_code`` holds the HTTP status when the @@ -265,7 +286,7 @@ def getPaths(urlOrPaths): def parseAndSave(option, urlOrPaths, outDir=None, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, responseMimeType='application/json', metaExtension='_meta.json', - services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta'}): + services={'meta': '/meta', 'text': '/tika/text', 'all': '/rmeta/xml'}): ''' Parse the objects and write extracted metadata and/or text in JSON format to matching filename with an extension of '_meta.json'. @@ -297,7 +318,7 @@ def parseAndSave(option, urlOrPaths, outDir=None, serverEndpoint=ServerEndpoint, def parse(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, responseMimeType='application/json', - services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta'}, rawResponse=False): + services={'meta': '/meta', 'text': '/tika/text', 'all': '/rmeta/xml'}, rawResponse=False): ''' Parse the objects and return extracted metadata and/or text in JSON format. :param option: @@ -314,7 +335,7 @@ def parse(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbose, ti def parse1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, responseMimeType='application/json', - services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/text'}, rawResponse=False, headers=None, config_path=None, requestOptions={}): + services={'meta': '/meta', 'text': '/tika/text', 'all': '/rmeta/text'}, rawResponse=False, headers=None, config_path=None, requestOptions={}): ''' Parse the object and return extracted metadata and/or text in JSON format. :param option: @@ -331,13 +352,13 @@ def parse1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, ti headers = headers or {} path, file_type = getRemoteFile(urlOrPath, TikaFilesPath) - headers.update({'Accept': responseMimeType, 'Content-Disposition': make_content_disposition_header(path.encode('utf-8') if type(path) is str else path)}) if option not in services: log.warning('config option must be one of meta, text, or all; using all.') service = services.get(option, services['all']) - if service == '/tika': responseMimeType = 'text/plain' - headers.update({'Accept': responseMimeType, 'Content-Disposition': make_content_disposition_header(path.encode('utf-8') if type(path) is str else path)}) + # Tika 4 selects the output format by path, not by Accept, and rejects a mismatching Accept with 406 + responseMimeType = _TikaAcceptTypes.get(service, responseMimeType) + headers.update({'Accept': responseMimeType, 'Content-Disposition': make_content_disposition_header(path)}) with urlOrPath if _is_file_object(urlOrPath) else open(path, 'rb') as f: status, response = callServer('put', serverEndpoint, service, f, headers, verbose, tikaServerJar, config_path=config_path, @@ -348,7 +369,7 @@ def parse1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, ti def detectLang(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, responseMimeType='text/plain', - services={'file' : '/language/stream'}): + services={'file' : '/language'}): ''' Detect the language of the provided stream and return its 2 character code as text/plain. :param option: @@ -366,9 +387,11 @@ def detectLang(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbos def detectLang1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, responseMimeType='text/plain', - services={'file' : '/language/stream'}, requestOptions={}): + services={'file' : '/language'}, requestOptions={}): ''' - Detect the language of the provided stream and return its 2 character code as text/plain. + Detect the language of the provided document and return its 2 character code as text/plain. + Since Tika 4 the /language endpoint expects text rather than a document, so the text is + extracted first (via /tika/text) and then submitted for detection. :param option: :param urlOrPath: :param serverEndpoint: @@ -378,12 +401,14 @@ def detectLang1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbos :param services: :return: ''' - path, mode = getRemoteFile(urlOrPath, TikaFilesPath) if option not in services: log.exception('Language option must be one of %s ' % bytes(services.keys())) raise TikaException('Language option must be one of %s ' % bytes(services.keys())) service = services[option] - status, response = callServer('put', serverEndpoint, service, open(path, 'rb'), + status, text = parse1('text', urlOrPath, serverEndpoint, verbose, tikaServerJar, requestOptions=requestOptions) + if status != 200 or not text: + return (status, text) + status, response = callServer('put', serverEndpoint, service, text, {'Accept': responseMimeType}, verbose, tikaServerJar, requestOptions=requestOptions) return (status, response) @@ -444,7 +469,7 @@ def doTranslate1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbo def detectType(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, responseMimeType='text/plain', - services={'type': '/detect/stream'}): + services={'type': '/detect'}): ''' Detect the MIME/media type of the stream and return it in text/plain. :param option: @@ -462,7 +487,7 @@ def detectType(option, urlOrPaths, serverEndpoint=ServerEndpoint, verbose=Verbos def detectType1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbose, tikaServerJar=TikaServerJar, responseMimeType='text/plain', - services={'type': '/detect/stream'}, config_path=None, requestOptions={}): + services={'type': '/detect'}, config_path=None, requestOptions={}): ''' Detect the MIME/media type of the stream and return it in text/plain. :param option: @@ -482,7 +507,7 @@ def detectType1(option, urlOrPath, serverEndpoint=ServerEndpoint, verbose=Verbos status, response = callServer('put', serverEndpoint, service, open(path, 'rb'), { 'Accept': responseMimeType, - 'Content-Disposition': make_content_disposition_header(path.encode('utf-8') if type(path) is str else path) + 'Content-Disposition': make_content_disposition_header(path) }, verbose, tikaServerJar, config_path=config_path, requestOptions=requestOptions) if csvOutput == 1: @@ -587,20 +612,12 @@ def checkTikaServer(scheme="http", serverHost=ServerHost, port=Port, tikaServerJ if port is None: port = '443' if scheme == 'https' else '80' - urlp = urlparse(tikaServerJar) serverEndpoint = '%s://%s:%s' % (scheme, serverHost, port) - jarPath = os.path.join(TikaJarPath, 'tika-server.jar') if 'localhost' in serverEndpoint or '127.0.0.1' in serverEndpoint: alreadyRunning = checkPortIsOpen(serverHost, port) if not alreadyRunning: - if not os.path.isfile(jarPath) and urlp.scheme != '': - getRemoteJar(tikaServerJar, jarPath) - - if not checkJarSig(tikaServerJar, jarPath): - os.remove(jarPath) - tikaServerJar = getRemoteJar(tikaServerJar, jarPath) - + jarPath = ensureTikaServerJar(tikaServerJar) status = startServer(jarPath, TikaJava, TikaJavaArgs, serverHost, port, classpath, config_path) if not status: log.error("Failed to receive startup confirmation from startServer.") @@ -680,20 +697,30 @@ def startServer(tikaServerJar, java_path = TikaJava, java_args = TikaJavaArgs, s log.error("Unable to create tika-server.log at %s due to permission error." % (TikaServerLogFilePath)) return False - # Check that specified java binary is available on path - try: - _ = Popen(java_path, stdout=open(os.devnull, "w"), stderr=open(os.devnull, "w")) - except FileNotFoundError as e: - log.error("Unable to run java; is it installed?") + # Check that the specified java binary is available and recent enough + javaVersion = _javaMajorVersion(java_path) + if javaVersion is None: + log.error("Unable to run java (%s); is it installed?", java_path) + return False + if javaVersion < TikaMinJavaVersion: + log.error("Apache Tika %s needs Java %d or later, found Java %d at %s", + TikaVersion, TikaMinJavaVersion, javaVersion, java_path) return False + # Tika 4 forks worker JVMs that resolve "java" from PATH; point them at the same runtime + env = os.environ.copy() + javaDir = os.path.dirname(java_path) + if javaDir: + env['PATH'] = javaDir + os.pathsep + env.get('PATH', '') + # Run java with jar args global TikaServerProcess + log.info("Starting Tika server: %s", cmd_string) # Patch for Windows support if Windows: - TikaServerProcess = Popen(cmd_string, stdout=logFile, stderr=STDOUT, shell=True, start_new_session=True) + TikaServerProcess = Popen(cmd_string, stdout=logFile, stderr=STDOUT, shell=True, start_new_session=True, env=env) else: - TikaServerProcess = Popen(cmd_string, stdout=logFile, stderr=STDOUT, shell=True, preexec_fn=os.setsid) + TikaServerProcess = Popen(cmd_string, stdout=logFile, stderr=STDOUT, shell=True, preexec_fn=os.setsid, env=env) # Check logs and retry as configured try_count = 0 @@ -711,10 +738,25 @@ def startServer(tikaServerJar, java_path = TikaJava, java_args = TikaJavaArgs, s if not is_started: log.error("Tika startup log message not received after %d tries." % (TikaStartupMaxRetry)) + _logServerLogTail(tika_log_file_path) return False else: return True + +def _logServerLogTail(logPath, lines=20): + ''' + Logs the last lines of the server log so a failed startup can be diagnosed without + hunting for tika-server.log. + ''' + try: + with open(logPath, 'r', errors='replace') as f: + tail = f.readlines()[-lines:] + except OSError: + return + if tail: + log.error("Last %d lines of %s:\n%s", len(tail), logPath, ''.join(tail).rstrip()) + def killServer(): ''' Kills the tika server started by the current execution instance diff --git a/tika/unpack.py b/tika/unpack.py index 01cb9fb1..eee81696 100644 --- a/tika/unpack.py +++ b/tika/unpack.py @@ -14,31 +14,62 @@ # limitations under the License. # -import csv import json import re -import tarfile import zipfile -from contextlib import closing -from io import BytesIO, TextIOWrapper +from io import BytesIO from .tika import ServerEndpoint, callServer, parse1 -_text_wrapper = TextIOWrapper - # Tika 4 /unpack/all output: a zip holding the original document as ``0.``, embedded # documents as ``1.``, ``2.``, ... and one ``.metadata.json`` sidecar per entry. +# The trailing handler name selects plain text for the extracted content (default: Markdown). UNPACK_SERVICE = '/unpack/all/text' _SIDECAR_SUFFIX = '.metadata.json' _CONTENT_KEY = 'tk:content' -def _parse_zip(zipOutput): +def from_file(filename, serverEndpoint=ServerEndpoint, requestOptions={}): + ''' + Parse from file + :param filename: file + :param serverEndpoint: Tika server end point (optional) + :return: dict with ``content`` and ``metadata`` of the document and ``attachments`` + mapping the name of each embedded document to its bytes; ``{}`` when the + server returned nothing + ''' + zipOutput = parse1('unpack', filename, serverEndpoint, + responseMimeType='application/zip', + services={'meta': '/meta', 'text': '/tika/text', + 'all': '/rmeta/xml', 'unpack': UNPACK_SERVICE}, + rawResponse=True, requestOptions=requestOptions) + return _parse(zipOutput) + + +def from_buffer(string, serverEndpoint=ServerEndpoint, headers=None, requestOptions={}): ''' - Turns a Tika 4 ``/unpack/all`` zip response into the same shape ``_parse`` produced for - the pre-4.0 tar output: ``content`` and ``metadata`` of the container document (taken - from its ``.metadata.json`` sidecar) and ``attachments`` mapping the name of each - embedded document to its bytes. Sidecars and the container itself are not attachments. + Parse from buffered content + :param string: buffered content + :param serverEndpoint: Tika server URL (Optional) + :return: parsed content, see ``from_file`` + ''' + + headers = headers or {} + headers.update({'Accept': 'application/zip'}) + + status, response = callServer('put', serverEndpoint, UNPACK_SERVICE, string, + headers, False, + rawResponse=True, requestOptions=requestOptions) + + return _parse((status, response)) + + +def _parse(zipOutput): + ''' + Turns a ``/unpack/all`` zip response into ``content`` and ``metadata`` of the container + document (taken from its ``.metadata.json`` sidecar) and ``attachments`` mapping the + name of each embedded document to its bytes. Sidecars and the container itself are + not attachments. :param zipOutput: ``(status, bytes)`` tuple as returned by ``callServer(rawResponse=True)`` :return: ``{}`` for an empty response, else a dict with ``content``, ``metadata``, ``attachments`` ''' @@ -80,99 +111,3 @@ def _sidecar_candidates(container, sidecars): candidates.append(stem + _SIDECAR_SUFFIX) candidates.append('0' + _SIDECAR_SUFFIX) return [c for c in candidates if c in sidecars] - - -def from_file(filename, serverEndpoint=ServerEndpoint, requestOptions={}): - ''' - Parse from file - :param filename: file - :param serverEndpoint: Tika server end point (optional) - :return: - ''' - tarOutput = parse1('unpack', filename, serverEndpoint, - responseMimeType='application/x-tar', - services={'meta': '/meta', 'text': '/tika', - 'all': '/rmeta/xml', 'unpack': '/unpack/all'}, - rawResponse=True, requestOptions=requestOptions) - return _parse(tarOutput) - - -def from_buffer(string, serverEndpoint=ServerEndpoint, headers=None, requestOptions={}): - ''' - Parse from buffered content - :param string: buffered content - :param serverEndpoint: Tika server URL (Optional) - :return: parsed content - ''' - - headers = headers or {} - headers.update({'Accept': 'application/x-tar'}) - - status, response = callServer('put', serverEndpoint, '/unpack/all', string, - headers, False, - rawResponse=True, requestOptions=requestOptions) - - return _parse((status, response)) - - -def _parse(tarOutput): - parsed = {} - if not tarOutput: - return parsed - elif tarOutput[1] is None or tarOutput[1] == b"": - return parsed - - with tarfile.open(fileobj=BytesIO(tarOutput[1])) as tarFile: - # get the member names - memberNames = list(tarFile.getnames()) - - # extract the metadata - metadata = {} - if "__METADATA__" in memberNames: - memberNames.remove("__METADATA__") - - metadataMember = tarFile.getmember("__METADATA__") - if not metadataMember.issym() and metadataMember.isfile(): - with closing(_text_wrapper(tarFile.extractfile(metadataMember), encoding=tarFile.encoding)) as metadataFile: - metadataReader = csv.reader(_truncate_nulls(metadataFile)) - for metadataLine in metadataReader: - # each metadata line comes as a key-value pair, with list values - # returned as extra values in the line - convert single values - # to non-list values to be consistent with parser metadata - assert len(metadataLine) >= 2 - - if len(metadataLine) > 2: - metadata[metadataLine[0]] = metadataLine[1:] - else: - metadata[metadataLine[0]] = metadataLine[1] - - - # get the content - content = "" - if "__TEXT__" in memberNames: - memberNames.remove("__TEXT__") - - contentMember = tarFile.getmember("__TEXT__") - if not contentMember.issym() and contentMember.isfile(): - with closing(_text_wrapper(tarFile.extractfile(contentMember), encoding='utf8')) as content_file: - content = content_file.read() - - # get the remaining files as attachments - attachments = {} - for attachment in memberNames: - attachmentMember = tarFile.getmember(attachment) - if not attachmentMember.issym() and attachmentMember.isfile(): - with closing(tarFile.extractfile(attachmentMember)) as attachment_file: - attachments[attachment] = attachment_file.read() - - parsed["content"] = content - parsed["metadata"] = metadata - parsed["attachments"] = attachments - - return parsed - - -# TODO: Remove if/when fixed. https://issues.apache.org/jira/browse/TIKA-3070 -def _truncate_nulls(s): - for line in s: - yield line.replace('\0', '') From 7467af698a047f6b0a36acf75920232e41cf38dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=B6rpel?= Date: Wed, 30 Sep 2026 18:10:23 +0200 Subject: [PATCH 3/4] feat: remove the translate interface --- README.md | 15 +-------- tika/tika.py | 78 ++++------------------------------------------- tika/translate.py | 66 --------------------------------------- 3 files changed, 7 insertions(+), 152 deletions(-) delete mode 100644 tika/translate.py diff --git a/README.md b/README.md index 3a5582ec..26abea80 100755 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ These are read once, when tika/tika.py is initially loaded and used throughout a 3. `TIKA_JAR_HASH_ALGO` - set to `sha1` when running on FIPS-compliant systems; default value is `md5`. 4. `TIKA_SERVER_ENDPOINT` - set to the host (local or remote) for the running Tika server jar. 5. `TIKA_CLIENT_ONLY` - if set to True, then `TIKA_SERVER_JAR` is ignored, and relies on the value for `TIKA_SERVER_ENDPOINT` and treats Tika like a REST client. -6. `TIKA_TRANSLATOR` - set to the fully qualified class name (defaults to Lingo24) for the Tika translator implementation. 7. `TIKA_SERVER_CLASSPATH` - set to a string (delimited by ':' for each additional path) to prepend to the Tika server jar path. 8. `TIKA_LOG_PATH` - set to a directory with write permissions and the `tika.log` and `tika-server.log` files will be placed in this directory. 9. `TIKA_PATH` - set to a directory with write permissions and the `tika_server.jar` file will be placed in this directory. @@ -187,16 +186,6 @@ from tika import language print(language.from_file('/path/to/file')) ``` -Translate Interface ------------------------- -The translate interface translates the text automatically extracted -by Tika from the source language to the destination language. - -```python -from tika import translate -print(translate.from_file('/path/to/spanish', 'es', 'en')) -``` - Using a Buffer -------------- Note you can also use a Parser and Detector @@ -253,7 +242,7 @@ print parsed["metadata"] Customizing the Tika Server Request --------------------------- -You may customize the outgoing HTTP request to Tika server by setting `requestOptions` on the `.from_file` and `.from_buffer` methods (Parser, Unpack , Detect, Config, Language, Translate). It should be a dictionary of arguments that will be passed to the request method. The [request method documentation](https://requests.kennethreitz.org/en/master/api/#requests.request) specifies valid arguments. This will override any defaults except for `url` and `params `/`data`. +You may customize the outgoing HTTP request to Tika server by setting `requestOptions` on the `.from_file` and `.from_buffer` methods (Parser, Unpack, Detect, Config, Language). It should be a dictionary of arguments that will be passed to the request method. The [request method documentation](https://requests.kennethreitz.org/en/master/api/#requests.request) specifies valid arguments. This will override any defaults except for `url` and `params `/`data`. ```python from tika import parser @@ -276,7 +265,6 @@ tika.py [-v] [-o ] [--server ] [--install ] [--server ] [--install ] [--port ]