From 0d045cbab19da5b28857167b15c257459dcd2091 Mon Sep 17 00:00:00 2001 From: zhangkun Date: Thu, 15 Oct 2026 20:26:53 +0800 Subject: [PATCH] fix(language_support): batch package queries per call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Replace the per-package dpkg-query and apt-cache processes with one dpkg status snapshot and one apt-cache policy batch per public call, because apt-cache loads the whole package cache and applies the pins on every start, which took seconds per candidate package. 2. Keep the snapshot local to the call, so an installation or an apt update is visible to the next query instead of returning stale results. 3. Mark name:arch entries as their plain name too and accept only the installed status, so a package installed for one architecture is no longer reported as missing. 4. Force LC_ALL=C for apt-cache, otherwise a translated apt prints localized field names and no candidate version can be parsed. 5. Bound each command with a context timeout, so a stalled tool cannot block the query forever. 6. Give each tool its own command function with a literal binary path, so the program to run is never taken from a variable. 7. Add tests covering the batching, includeInstalled, the GNOME and dictionary exclusions, query failures and the snapshot refresh. Log: One language package query now runs a single dpkg-query and apt-cache. Influence: Language package queries no longer time out. fix(language_support): 语言包查询改为按调用批量执行 1. 不再逐包启动 dpkg-query 和 apt-cache，改为每次公开调用各执行一次 dpkg 状态读取和一次 apt-cache policy 批量查询；apt-cache 每次启动都要加载整个包缓存并计算 pin，单个候选包就要数秒。 2. 快照只在本次调用内有效，安装或 apt 更新后下一次查询即可看到新状态，不会返回过期结果。 3. name:arch 条目同时标记其原始包名，且只接受 installed 状态，某一架构已安装的包不再被判定为缺失。 4. apt-cache 强制使用 LC_ALL=C，否则翻译过的 apt 会输出本地化字段名，解析不到候选版本。 5. 每次调用加 context 超时，避免工具卡住时查询被无限期阻塞。 6. 两个工具各自使用独立的命令函数并写死二进制绝对路径，执行哪个程序不取自变量。 7. 补充测试，覆盖批量查询、includeInstalled、GNOME 与词典排除规则、查询失败以及快照刷新。 Log: 一次语言包查询现在只执行一次 dpkg-query 和一次 apt-cache。 PMS: BUG-379697 PMS: BUG-379687 PMS: BUG-379685 Influence: 语言包查询不再超时。 --- language_support/lang_support.go | 201 +++++++++++-------- language_support/lang_support_test.go | 269 +++++++++++++++++++++++++- 2 files changed, 391 insertions(+), 79 deletions(-) diff --git a/language_support/lang_support.go b/language_support/lang_support.go index 5b462ac..1da4463 100644 --- a/language_support/lang_support.go +++ b/language_support/lang_support.go @@ -1,16 +1,17 @@ -// SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. +// SPDX-FileCopyrightText: 2022 - 2026 UnionTech Software Technology Co., Ltd. // // SPDX-License-Identifier: GPL-3.0-or-later package language_support import ( - "bytes" + "context" "encoding/csv" "io" "os" "os/exec" "strings" + "time" libLocale "github.com/linuxdeepin/go-lib/locale" ) @@ -35,84 +36,136 @@ func NewLanguageSupport() (ls *LanguageSupport, err error) { func (ls *LanguageSupport) Destroy() { } -func (ls *LanguageSupport) isPkgInstalled(name string) (bool, error) { - output, err := exec.Command("/usr/bin/dpkg-query", "-W", "-f", - "${db:Status-Status}", "--", name).Output() - if err != nil { - return false, err - } - return bytes.Equal(bytes.TrimSpace(output), []byte("installed")), nil +// commandTimeout bounds a single dpkg-query or apt-cache run. +const commandTimeout = 30 * time.Second + +// dpkgQueryCommand and aptCacheCommand are variables so that tests can supply +// canned output. +var dpkgQueryCommand = func(args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + return exec.CommandContext(ctx, "/usr/bin/dpkg-query", args...).Output() } -func (ls *LanguageSupport) isPkgInstallable(name string) (bool, error) { - cmd := exec.Command("/usr/bin/apt-cache", "policy", - "--", name) - cmd.Env = []string{"LANG=en_US.UTF-8"} - output, err := cmd.Output() - if err != nil { - return false, err +var aptCacheCommand = func(args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "/usr/bin/apt-cache", args...) + // Only LC_ALL=C: a translated apt prints localized field names. + cmd.Env = []string{"LC_ALL=C"} + return cmd.Output() +} + +// pkgQuery is the dpkg/apt snapshot of a single call: one dpkg-query and one +// apt-cache process, not cached across calls. +type pkgQuery struct { + installed map[string]bool + candidates map[string]bool +} + +func newPkgQuery() *pkgQuery { + return &pkgQuery{candidates: make(map[string]bool)} +} + +// isInstalled reads the whole dpkg database once; a multi-arch entry +// ("name:arch") also marks the plain name. +func (q *pkgQuery) isInstalled(name string) bool { + if q.installed == nil { + q.installed = make(map[string]bool) + output, err := dpkgQueryCommand("-W", "-f", + "${binary:Package} ${db:Status-Status}\n") + if err == nil { + for _, line := range strings.Split(string(output), "\n") { + pkg, status, found := strings.Cut(line, " ") + if !found || status != "installed" { + continue + } + q.installed[pkg] = true + if i := strings.IndexByte(pkg, ':'); i > 0 { + q.installed[pkg[:i]] = true + } + } + } } + return q.installed[name] +} - const candidate = "Candidate: " - idx := bytes.Index(output, []byte(candidate)) - if idx == -1 { - return false, nil +// loadCandidates asks apt once for all given names; unknown names and +// "(none)" candidates stay unmarked. +func (q *pkgQuery) loadCandidates(names []string) { + if len(names) == 0 { + return + } + args := append([]string{"policy", "--"}, names...) + output, err := aptCacheCommand(args...) + if err != nil { + return } - if bytes.HasPrefix(output[idx+len(candidate):], []byte("(none)")) { - return false, nil + var pkg string + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + if line[0] != ' ' && strings.HasSuffix(line, ":") { // "package:" header + pkg = strings.TrimSuffix(line, ":") + continue + } + version, found := strings.CutPrefix(strings.TrimSpace(line), "Candidate:") + if !found || pkg == "" { + continue + } + version = strings.TrimSpace(version) + q.candidates[pkg] = version != "" && version != "(none)" } - return true, nil } -// ByPackageAndLocale get language support packages for a package and locale. -func (ls *LanguageSupport) ByPackageAndLocale( - package0 string, locale string, includeInstalled bool) (packages []string) { - - packagesTemp := make(map[string]struct{}) +// candidateNames lists the packages offered for one trigger and locale. +func (ls *LanguageSupport) candidateNames(package0, locale string) (names []string) { depMap := ls.pkgDepends[package0] // check explicit entries for that locale for _, pkgs := range depMap[langCodeFromLocale(locale)] { - for _, pkg := range pkgs { - installable, err := ls.isPkgInstallable(pkg) - if err != nil { - continue - } - if installable { - packagesTemp[pkg] = struct{}{} - } - } + names = append(names, pkgs...) } // check patterns for empty locale string (i. e. applies to any locale) for _, patternList := range depMap[""] { for _, pattern := range patternList { - pkgs := expendPkgPattern(pattern, locale) - for _, pkg := range pkgs { - installable, err := ls.isPkgInstallable(pkg) - if err != nil { - continue - } - if installable { - packagesTemp[pkg] = struct{}{} - } - } + names = append(names, expendPkgPattern(pattern, locale)...) } } + return +} - if !includeInstalled { // not show installed - for pkg := range packagesTemp { - isInstalled, err := ls.isPkgInstalled(pkg) - if err != nil { +// packagesFor batches the candidates of all given triggers and applies the +// filtering rules to the result. +func (ls *LanguageSupport) packagesFor( + triggers []string, locale string, includeInstalled bool, q *pkgQuery) []string { + + names := make([]string, 0) + seen := make(map[string]struct{}) + for _, trigger := range triggers { + for _, name := range ls.candidateNames(trigger, locale) { + if _, dup := seen[name]; dup { continue } + seen[name] = struct{}{} + names = append(names, name) + } + } - // filter out installed packages - if isInstalled { - delete(packagesTemp, pkg) - } + q.loadCandidates(names) + + packagesTemp := make(map[string]struct{}) + for _, name := range names { + if !q.candidates[name] { + continue } + if !includeInstalled && q.isInstalled(name) { // not show installed + continue + } + packagesTemp[name] = struct{}{} } // exclude Fcitx packages if GNOME desktop @@ -142,40 +195,32 @@ func (ls *LanguageSupport) ByPackageAndLocale( // https://launchpad.net/bugs/1578821 delete(packagesTemp, "hunspell-gl") - packages = make([]string, 0, len(packagesTemp)) + packages := make([]string, 0, len(packagesTemp)) for pkg := range packagesTemp { packages = append(packages, pkg) } - return + return packages +} + +// ByPackageAndLocale get language support packages for a package and locale. +func (ls *LanguageSupport) ByPackageAndLocale( + package0 string, locale string, includeInstalled bool) (packages []string) { + + return ls.packagesFor([]string{package0}, locale, includeInstalled, newPkgQuery()) } // ByLocale get language support packages for a locale func (ls *LanguageSupport) ByLocale(locale string, includeInstalled bool) []string { - packagesTemp := make(map[string]struct{}) - for trigger := range ls.pkgDepends { - var add bool - if trigger == "" { - add = true - } else { - pkgInstalled, err := ls.isPkgInstalled(trigger) - if err == nil && pkgInstalled { - add = true - } - } + q := newPkgQuery() - if add { - pkgs := ls.ByPackageAndLocale(trigger, locale, includeInstalled) - for _, pkg := range pkgs { - packagesTemp[pkg] = struct{}{} - } + var triggers []string + for trigger := range ls.pkgDepends { + if trigger == "" || q.isInstalled(trigger) { + triggers = append(triggers, trigger) } } - packages := make([]string, 0, len(packagesTemp)) - for pkg := range packagesTemp { - packages = append(packages, pkg) - } - return packages + return ls.packagesFor(triggers, locale, includeInstalled, q) } func expendPkgPattern(pattern, locale string) (pkgs []string) { diff --git a/language_support/lang_support_test.go b/language_support/lang_support_test.go index 94c1090..7c613a5 100644 --- a/language_support/lang_support_test.go +++ b/language_support/lang_support_test.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. +// SPDX-FileCopyrightText: 2022 - 2026 UnionTech Software Technology Co., Ltd. // // SPDX-License-Identifier: GPL-3.0-or-later @@ -6,6 +6,8 @@ package language_support import ( "encoding/json" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -65,3 +67,268 @@ func TestExpendPkgPattern(t *testing.T) { pkgs = expendPkgPattern("[p]", "wa_BE@euro") assert.ElementsMatch(t, pkgs, []string{"[p]", "[p]wa", "[p]wabe", "[p]wa-be", "[p]wa-euro", "[p]wa-be-euro"}) } + +type commandCall struct { + name string + args []string +} + +type fakeCommands struct { + calls []commandCall + dpkgOut string + dpkgErr error + aptOut string + aptErr error +} + +const ( + dpkgCmdName = "dpkg-query" + aptCmdName = "apt-cache" +) + +// install stubs both commands with the canned output and records each call. +func (f *fakeCommands) install(t *testing.T) { + t.Helper() + prevDpkg, prevApt := dpkgQueryCommand, aptCacheCommand + t.Cleanup(func() { + dpkgQueryCommand = prevDpkg + aptCacheCommand = prevApt + }) + dpkgQueryCommand = func(args ...string) ([]byte, error) { + f.calls = append(f.calls, commandCall{name: dpkgCmdName, args: args}) + return []byte(f.dpkgOut), f.dpkgErr + } + aptCacheCommand = func(args ...string) ([]byte, error) { + f.calls = append(f.calls, commandCall{name: aptCmdName, args: args}) + return []byte(f.aptOut), f.aptErr + } +} + +func (f *fakeCommands) callsOf(name string) (out []commandCall) { + for _, call := range f.calls { + if call.name == name { + out = append(out, call) + } + } + return +} + +// testLanguageSupport builds a LanguageSupport from pkg_depends content. +func testLanguageSupport(t *testing.T, depends string) *LanguageSupport { + t.Helper() + path := filepath.Join(t.TempDir(), "pkg_depends") + assert.NoError(t, os.WriteFile(path, []byte(depends), 0644)) + pkgDepends, err := parsePkgDepends(path) + assert.NoError(t, err) + return &LanguageSupport{pkgDepends: pkgDepends} +} + +const batchDepends = `tr::firefox:firefox-locale- +tr::thunderbird:thunderbird-locale- +tr::libreoffice-common:hunspell- +wa:de:libreoffice-common:hunspell-de-at-frami +im:de::fcitx +im:de::fcitx-frontend-gtk3 +` + +// dpkg-query -W output of a machine with a native and a foreign arch entry for +// the same package, an installed trigger, a removed trigger and an installed +// package that must stay excluded by the dictionary rule. +const batchDpkgOut = `firefox:amd64 installed +firefox-locale-de:amd64 installed +firefox-locale-de:i386 installed +thunderbird:amd64 config-files +libreoffice-common:amd64 installed +hunspell-de-de:amd64 installed +hunspell-de-de:i386 installed +` + +// apt-cache policy output for one batch: "(none)" candidates, a package with no +// block at all, and an unrequested foreign-arch block that must not leak in. +const batchAptOut = `firefox-locale-: + Installed: (none) + Candidate: (none) +firefox-locale-de: + Installed: (none) + Candidate: 1.0 +firefox-locale-dede: + Installed: (none) + Candidate: (none) +firefox-locale-de-de: + Installed: (none) + Candidate: 1.0 +hunspell-de-at-frami: + Installed: (none) + Candidate: 1.0 +hunspell-: + Installed: (none) + Candidate: (none) +hunspell-de: + Installed: (none) + Candidate: 1.0 +hunspell-dede: + Installed: (none) + Candidate: (none) +hunspell-de-de: + Installed: (none) + Candidate: 1.0 +fcitx: + Installed: (none) + Candidate: 1.0 +fcitx-frontend-gtk3: + Installed: (none) + Candidate: 1.0 +libc6:i386: + Installed: (none) + Candidate: 9.9 +` + +var batchCandidates = []string{ + "firefox-locale-", "firefox-locale-de", "firefox-locale-dede", "firefox-locale-de-de", + "hunspell-", "hunspell-de", "hunspell-dede", "hunspell-de-de", "hunspell-de-at-frami", + "fcitx", "fcitx-frontend-gtk3", +} + +func TestByLocaleBatchesQueries(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + fake := &fakeCommands{dpkgOut: batchDpkgOut, aptOut: batchAptOut} + fake.install(t) + ls := testLanguageSupport(t, batchDepends) + + packages := ls.ByLocale("de_DE", false) + + assert.ElementsMatch(t, []string{ + "firefox-locale-de-de", "hunspell-de", "hunspell-de-at-frami", + "fcitx", "fcitx-frontend-gtk3", + }, packages) + + // one dpkg snapshot and one apt batch per call, no per-package processes + dpkgCalls := fake.callsOf(dpkgCmdName) + aptCalls := fake.callsOf(aptCmdName) + assert.Len(t, dpkgCalls, 1) + assert.Len(t, aptCalls, 1) + assert.Equal(t, []string{"-W", "-f", "${binary:Package} ${db:Status-Status}\n"}, dpkgCalls[0].args) + + // every candidate is asked for exactly once, in a single apt invocation + assert.Equal(t, "policy", aptCalls[0].args[0]) + assert.Equal(t, "--", aptCalls[0].args[1]) + assert.ElementsMatch(t, batchCandidates, aptCalls[0].args[2:]) + assert.Len(t, aptCalls[0].args[2:], len(batchCandidates)) +} + +func TestByLocaleIncludeInstalled(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + fake := &fakeCommands{dpkgOut: batchDpkgOut, aptOut: batchAptOut} + fake.install(t) + ls := testLanguageSupport(t, batchDepends) + + packages := ls.ByLocale("de_DE", true) + + // installed packages show up again, hunspell-de-de stays out (conflicts + // with hunspell-de-de-frami) + assert.ElementsMatch(t, []string{ + "firefox-locale-de", "firefox-locale-de-de", "hunspell-de", + "hunspell-de-at-frami", "fcitx", "fcitx-frontend-gtk3", + }, packages) +} + +func TestByLocaleGnomeExcludesFcitx(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "GNOME") + fake := &fakeCommands{dpkgOut: batchDpkgOut, aptOut: batchAptOut} + fake.install(t) + ls := testLanguageSupport(t, batchDepends) + + packages := ls.ByLocale("de_DE", true) + + assert.ElementsMatch(t, []string{ + "firefox-locale-de", "firefox-locale-de-de", "hunspell-de", "hunspell-de-at-frami", + }, packages) +} + +func TestByPackageAndLocaleBatchesCandidates(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + // duplicated entries must not turn into duplicated apt arguments + ls := testLanguageSupport(t, `tr::libreoffice-common:hunspell- +wa:de:libreoffice-common:hunspell-de-at-frami +wa:de:libreoffice-common:hunspell-de-at-frami +`) + + fake := &fakeCommands{ + dpkgOut: "hunspell-de-at-frami:amd64 installed\n", + aptOut: `hunspell-: + Installed: (none) + Candidate: (none) +hunspell-de: + Installed: (none) + Candidate: 1.0 +hunspell-dede: + Installed: (none) + Candidate: (none) +hunspell-de-at-frami: + Installed: (none) + Candidate: 1.0 +`, + } + fake.install(t) + + packages := ls.ByPackageAndLocale("libreoffice-common", "de_DE", false) + assert.ElementsMatch(t, []string{"hunspell-de"}, packages) + + aptCalls := fake.callsOf(aptCmdName) + assert.Len(t, aptCalls, 1) + assert.ElementsMatch(t, []string{ + "hunspell-", "hunspell-de", "hunspell-dede", "hunspell-de-de", "hunspell-de-at-frami", + }, aptCalls[0].args[2:]) + assert.Len(t, aptCalls[0].args[2:], 5) // no duplicate argument + + // installed package kept when asked for + fake.calls = nil + packages = ls.ByPackageAndLocale("libreoffice-common", "de_DE", true) + assert.ElementsMatch(t, []string{"hunspell-de", "hunspell-de-at-frami"}, packages) + assert.Len(t, fake.callsOf(aptCmdName), 1) +} + +func TestQueryFailuresDoNotPanicAndKeepOldFiltering(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + ls := testLanguageSupport(t, `tr::firefox:firefox-locale- +wa:de::hunspell-de-at-frami +`) + + // dpkg fails: no trigger counts as installed, so only the empty trigger + // contributes, and the failing command is not retried per package + fake := &fakeCommands{ + dpkgErr: assert.AnError, + aptOut: `firefox-locale-de: + Installed: (none) + Candidate: 1.0 +hunspell-de-at-frami: + Installed: (none) + Candidate: 1.0 +`, + } + fake.install(t) + assert.ElementsMatch(t, []string{"hunspell-de-at-frami"}, ls.ByLocale("de_DE", false)) + assert.Len(t, fake.callsOf(dpkgCmdName), 1) + + // apt fails: nothing is installable + fake.calls = nil + fake.dpkgErr = nil + fake.aptErr = assert.AnError + assert.Empty(t, ls.ByLocale("de_DE", false)) + assert.Len(t, fake.callsOf(aptCmdName), 1) +} + +func TestLanguageSupportRefreshesPackageState(t *testing.T) { + ls := testLanguageSupport(t, "fn:ja::fonts-example\n") + fake := &fakeCommands{aptOut: "fonts-example:\n Candidate: 1.0\n"} + fake.install(t) + assert.ElementsMatch(t, []string{"fonts-example"}, ls.ByLocale("ja_JP", false)) + + // Reusing the object after installation must not return stale missing packages. + fake.dpkgOut = "fonts-example installed\n" + assert.Empty(t, ls.ByLocale("ja_JP", false)) + + // Repository changes must also be visible to the next query. + fake.aptOut = "fonts-example:\n Candidate: (none)\n" + assert.Empty(t, ls.ByLocale("ja_JP", true)) +}