diff --git a/language_support/lang_support.go b/language_support/lang_support.go index 5b462ac..1da4463 100644 --- a/language_support/lang_support.go +++ b/language_support/lang_support.go @@ -1,16 +1,17 @@ -// SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. +// SPDX-FileCopyrightText: 2022 - 2026 UnionTech Software Technology Co., Ltd. // // SPDX-License-Identifier: GPL-3.0-or-later package language_support import ( - "bytes" + "context" "encoding/csv" "io" "os" "os/exec" "strings" + "time" libLocale "github.com/linuxdeepin/go-lib/locale" ) @@ -35,84 +36,136 @@ func NewLanguageSupport() (ls *LanguageSupport, err error) { func (ls *LanguageSupport) Destroy() { } -func (ls *LanguageSupport) isPkgInstalled(name string) (bool, error) { - output, err := exec.Command("/usr/bin/dpkg-query", "-W", "-f", - "${db:Status-Status}", "--", name).Output() - if err != nil { - return false, err - } - return bytes.Equal(bytes.TrimSpace(output), []byte("installed")), nil +// commandTimeout bounds a single dpkg-query or apt-cache run. +const commandTimeout = 30 * time.Second + +// dpkgQueryCommand and aptCacheCommand are variables so that tests can supply +// canned output. +var dpkgQueryCommand = func(args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + return exec.CommandContext(ctx, "/usr/bin/dpkg-query", args...).Output() } -func (ls *LanguageSupport) isPkgInstallable(name string) (bool, error) { - cmd := exec.Command("/usr/bin/apt-cache", "policy", - "--", name) - cmd.Env = []string{"LANG=en_US.UTF-8"} - output, err := cmd.Output() - if err != nil { - return false, err +var aptCacheCommand = func(args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "/usr/bin/apt-cache", args...) + // Only LC_ALL=C: a translated apt prints localized field names. + cmd.Env = []string{"LC_ALL=C"} + return cmd.Output() +} + +// pkgQuery is the dpkg/apt snapshot of a single call: one dpkg-query and one +// apt-cache process, not cached across calls. +type pkgQuery struct { + installed map[string]bool + candidates map[string]bool +} + +func newPkgQuery() *pkgQuery { + return &pkgQuery{candidates: make(map[string]bool)} +} + +// isInstalled reads the whole dpkg database once; a multi-arch entry +// ("name:arch") also marks the plain name. +func (q *pkgQuery) isInstalled(name string) bool { + if q.installed == nil { + q.installed = make(map[string]bool) + output, err := dpkgQueryCommand("-W", "-f", + "${binary:Package} ${db:Status-Status}\n") + if err == nil { + for _, line := range strings.Split(string(output), "\n") { + pkg, status, found := strings.Cut(line, " ") + if !found || status != "installed" { + continue + } + q.installed[pkg] = true + if i := strings.IndexByte(pkg, ':'); i > 0 { + q.installed[pkg[:i]] = true + } + } + } } + return q.installed[name] +} - const candidate = "Candidate: " - idx := bytes.Index(output, []byte(candidate)) - if idx == -1 { - return false, nil +// loadCandidates asks apt once for all given names; unknown names and +// "(none)" candidates stay unmarked. +func (q *pkgQuery) loadCandidates(names []string) { + if len(names) == 0 { + return + } + args := append([]string{"policy", "--"}, names...) + output, err := aptCacheCommand(args...) + if err != nil { + return } - if bytes.HasPrefix(output[idx+len(candidate):], []byte("(none)")) { - return false, nil + var pkg string + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + if line[0] != ' ' && strings.HasSuffix(line, ":") { // "package:" header + pkg = strings.TrimSuffix(line, ":") + continue + } + version, found := strings.CutPrefix(strings.TrimSpace(line), "Candidate:") + if !found || pkg == "" { + continue + } + version = strings.TrimSpace(version) + q.candidates[pkg] = version != "" && version != "(none)" } - return true, nil } -// ByPackageAndLocale get language support packages for a package and locale. -func (ls *LanguageSupport) ByPackageAndLocale( - package0 string, locale string, includeInstalled bool) (packages []string) { - - packagesTemp := make(map[string]struct{}) +// candidateNames lists the packages offered for one trigger and locale. +func (ls *LanguageSupport) candidateNames(package0, locale string) (names []string) { depMap := ls.pkgDepends[package0] // check explicit entries for that locale for _, pkgs := range depMap[langCodeFromLocale(locale)] { - for _, pkg := range pkgs { - installable, err := ls.isPkgInstallable(pkg) - if err != nil { - continue - } - if installable { - packagesTemp[pkg] = struct{}{} - } - } + names = append(names, pkgs...) } // check patterns for empty locale string (i. e. applies to any locale) for _, patternList := range depMap[""] { for _, pattern := range patternList { - pkgs := expendPkgPattern(pattern, locale) - for _, pkg := range pkgs { - installable, err := ls.isPkgInstallable(pkg) - if err != nil { - continue - } - if installable { - packagesTemp[pkg] = struct{}{} - } - } + names = append(names, expendPkgPattern(pattern, locale)...) } } + return +} - if !includeInstalled { // not show installed - for pkg := range packagesTemp { - isInstalled, err := ls.isPkgInstalled(pkg) - if err != nil { +// packagesFor batches the candidates of all given triggers and applies the +// filtering rules to the result. +func (ls *LanguageSupport) packagesFor( + triggers []string, locale string, includeInstalled bool, q *pkgQuery) []string { + + names := make([]string, 0) + seen := make(map[string]struct{}) + for _, trigger := range triggers { + for _, name := range ls.candidateNames(trigger, locale) { + if _, dup := seen[name]; dup { continue } + seen[name] = struct{}{} + names = append(names, name) + } + } - // filter out installed packages - if isInstalled { - delete(packagesTemp, pkg) - } + q.loadCandidates(names) + + packagesTemp := make(map[string]struct{}) + for _, name := range names { + if !q.candidates[name] { + continue } + if !includeInstalled && q.isInstalled(name) { // not show installed + continue + } + packagesTemp[name] = struct{}{} } // exclude Fcitx packages if GNOME desktop @@ -142,40 +195,32 @@ func (ls *LanguageSupport) ByPackageAndLocale( // https://launchpad.net/bugs/1578821 delete(packagesTemp, "hunspell-gl") - packages = make([]string, 0, len(packagesTemp)) + packages := make([]string, 0, len(packagesTemp)) for pkg := range packagesTemp { packages = append(packages, pkg) } - return + return packages +} + +// ByPackageAndLocale get language support packages for a package and locale. +func (ls *LanguageSupport) ByPackageAndLocale( + package0 string, locale string, includeInstalled bool) (packages []string) { + + return ls.packagesFor([]string{package0}, locale, includeInstalled, newPkgQuery()) } // ByLocale get language support packages for a locale func (ls *LanguageSupport) ByLocale(locale string, includeInstalled bool) []string { - packagesTemp := make(map[string]struct{}) - for trigger := range ls.pkgDepends { - var add bool - if trigger == "" { - add = true - } else { - pkgInstalled, err := ls.isPkgInstalled(trigger) - if err == nil && pkgInstalled { - add = true - } - } + q := newPkgQuery() - if add { - pkgs := ls.ByPackageAndLocale(trigger, locale, includeInstalled) - for _, pkg := range pkgs { - packagesTemp[pkg] = struct{}{} - } + var triggers []string + for trigger := range ls.pkgDepends { + if trigger == "" || q.isInstalled(trigger) { + triggers = append(triggers, trigger) } } - packages := make([]string, 0, len(packagesTemp)) - for pkg := range packagesTemp { - packages = append(packages, pkg) - } - return packages + return ls.packagesFor(triggers, locale, includeInstalled, q) } func expendPkgPattern(pattern, locale string) (pkgs []string) { diff --git a/language_support/lang_support_test.go b/language_support/lang_support_test.go index 94c1090..7c613a5 100644 --- a/language_support/lang_support_test.go +++ b/language_support/lang_support_test.go @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. +// SPDX-FileCopyrightText: 2022 - 2026 UnionTech Software Technology Co., Ltd. // // SPDX-License-Identifier: GPL-3.0-or-later @@ -6,6 +6,8 @@ package language_support import ( "encoding/json" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -65,3 +67,268 @@ func TestExpendPkgPattern(t *testing.T) { pkgs = expendPkgPattern("[p]", "wa_BE@euro") assert.ElementsMatch(t, pkgs, []string{"[p]", "[p]wa", "[p]wabe", "[p]wa-be", "[p]wa-euro", "[p]wa-be-euro"}) } + +type commandCall struct { + name string + args []string +} + +type fakeCommands struct { + calls []commandCall + dpkgOut string + dpkgErr error + aptOut string + aptErr error +} + +const ( + dpkgCmdName = "dpkg-query" + aptCmdName = "apt-cache" +) + +// install stubs both commands with the canned output and records each call. +func (f *fakeCommands) install(t *testing.T) { + t.Helper() + prevDpkg, prevApt := dpkgQueryCommand, aptCacheCommand + t.Cleanup(func() { + dpkgQueryCommand = prevDpkg + aptCacheCommand = prevApt + }) + dpkgQueryCommand = func(args ...string) ([]byte, error) { + f.calls = append(f.calls, commandCall{name: dpkgCmdName, args: args}) + return []byte(f.dpkgOut), f.dpkgErr + } + aptCacheCommand = func(args ...string) ([]byte, error) { + f.calls = append(f.calls, commandCall{name: aptCmdName, args: args}) + return []byte(f.aptOut), f.aptErr + } +} + +func (f *fakeCommands) callsOf(name string) (out []commandCall) { + for _, call := range f.calls { + if call.name == name { + out = append(out, call) + } + } + return +} + +// testLanguageSupport builds a LanguageSupport from pkg_depends content. +func testLanguageSupport(t *testing.T, depends string) *LanguageSupport { + t.Helper() + path := filepath.Join(t.TempDir(), "pkg_depends") + assert.NoError(t, os.WriteFile(path, []byte(depends), 0644)) + pkgDepends, err := parsePkgDepends(path) + assert.NoError(t, err) + return &LanguageSupport{pkgDepends: pkgDepends} +} + +const batchDepends = `tr::firefox:firefox-locale- +tr::thunderbird:thunderbird-locale- +tr::libreoffice-common:hunspell- +wa:de:libreoffice-common:hunspell-de-at-frami +im:de::fcitx +im:de::fcitx-frontend-gtk3 +` + +// dpkg-query -W output of a machine with a native and a foreign arch entry for +// the same package, an installed trigger, a removed trigger and an installed +// package that must stay excluded by the dictionary rule. +const batchDpkgOut = `firefox:amd64 installed +firefox-locale-de:amd64 installed +firefox-locale-de:i386 installed +thunderbird:amd64 config-files +libreoffice-common:amd64 installed +hunspell-de-de:amd64 installed +hunspell-de-de:i386 installed +` + +// apt-cache policy output for one batch: "(none)" candidates, a package with no +// block at all, and an unrequested foreign-arch block that must not leak in. +const batchAptOut = `firefox-locale-: + Installed: (none) + Candidate: (none) +firefox-locale-de: + Installed: (none) + Candidate: 1.0 +firefox-locale-dede: + Installed: (none) + Candidate: (none) +firefox-locale-de-de: + Installed: (none) + Candidate: 1.0 +hunspell-de-at-frami: + Installed: (none) + Candidate: 1.0 +hunspell-: + Installed: (none) + Candidate: (none) +hunspell-de: + Installed: (none) + Candidate: 1.0 +hunspell-dede: + Installed: (none) + Candidate: (none) +hunspell-de-de: + Installed: (none) + Candidate: 1.0 +fcitx: + Installed: (none) + Candidate: 1.0 +fcitx-frontend-gtk3: + Installed: (none) + Candidate: 1.0 +libc6:i386: + Installed: (none) + Candidate: 9.9 +` + +var batchCandidates = []string{ + "firefox-locale-", "firefox-locale-de", "firefox-locale-dede", "firefox-locale-de-de", + "hunspell-", "hunspell-de", "hunspell-dede", "hunspell-de-de", "hunspell-de-at-frami", + "fcitx", "fcitx-frontend-gtk3", +} + +func TestByLocaleBatchesQueries(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + fake := &fakeCommands{dpkgOut: batchDpkgOut, aptOut: batchAptOut} + fake.install(t) + ls := testLanguageSupport(t, batchDepends) + + packages := ls.ByLocale("de_DE", false) + + assert.ElementsMatch(t, []string{ + "firefox-locale-de-de", "hunspell-de", "hunspell-de-at-frami", + "fcitx", "fcitx-frontend-gtk3", + }, packages) + + // one dpkg snapshot and one apt batch per call, no per-package processes + dpkgCalls := fake.callsOf(dpkgCmdName) + aptCalls := fake.callsOf(aptCmdName) + assert.Len(t, dpkgCalls, 1) + assert.Len(t, aptCalls, 1) + assert.Equal(t, []string{"-W", "-f", "${binary:Package} ${db:Status-Status}\n"}, dpkgCalls[0].args) + + // every candidate is asked for exactly once, in a single apt invocation + assert.Equal(t, "policy", aptCalls[0].args[0]) + assert.Equal(t, "--", aptCalls[0].args[1]) + assert.ElementsMatch(t, batchCandidates, aptCalls[0].args[2:]) + assert.Len(t, aptCalls[0].args[2:], len(batchCandidates)) +} + +func TestByLocaleIncludeInstalled(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + fake := &fakeCommands{dpkgOut: batchDpkgOut, aptOut: batchAptOut} + fake.install(t) + ls := testLanguageSupport(t, batchDepends) + + packages := ls.ByLocale("de_DE", true) + + // installed packages show up again, hunspell-de-de stays out (conflicts + // with hunspell-de-de-frami) + assert.ElementsMatch(t, []string{ + "firefox-locale-de", "firefox-locale-de-de", "hunspell-de", + "hunspell-de-at-frami", "fcitx", "fcitx-frontend-gtk3", + }, packages) +} + +func TestByLocaleGnomeExcludesFcitx(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "GNOME") + fake := &fakeCommands{dpkgOut: batchDpkgOut, aptOut: batchAptOut} + fake.install(t) + ls := testLanguageSupport(t, batchDepends) + + packages := ls.ByLocale("de_DE", true) + + assert.ElementsMatch(t, []string{ + "firefox-locale-de", "firefox-locale-de-de", "hunspell-de", "hunspell-de-at-frami", + }, packages) +} + +func TestByPackageAndLocaleBatchesCandidates(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + // duplicated entries must not turn into duplicated apt arguments + ls := testLanguageSupport(t, `tr::libreoffice-common:hunspell- +wa:de:libreoffice-common:hunspell-de-at-frami +wa:de:libreoffice-common:hunspell-de-at-frami +`) + + fake := &fakeCommands{ + dpkgOut: "hunspell-de-at-frami:amd64 installed\n", + aptOut: `hunspell-: + Installed: (none) + Candidate: (none) +hunspell-de: + Installed: (none) + Candidate: 1.0 +hunspell-dede: + Installed: (none) + Candidate: (none) +hunspell-de-at-frami: + Installed: (none) + Candidate: 1.0 +`, + } + fake.install(t) + + packages := ls.ByPackageAndLocale("libreoffice-common", "de_DE", false) + assert.ElementsMatch(t, []string{"hunspell-de"}, packages) + + aptCalls := fake.callsOf(aptCmdName) + assert.Len(t, aptCalls, 1) + assert.ElementsMatch(t, []string{ + "hunspell-", "hunspell-de", "hunspell-dede", "hunspell-de-de", "hunspell-de-at-frami", + }, aptCalls[0].args[2:]) + assert.Len(t, aptCalls[0].args[2:], 5) // no duplicate argument + + // installed package kept when asked for + fake.calls = nil + packages = ls.ByPackageAndLocale("libreoffice-common", "de_DE", true) + assert.ElementsMatch(t, []string{"hunspell-de", "hunspell-de-at-frami"}, packages) + assert.Len(t, fake.callsOf(aptCmdName), 1) +} + +func TestQueryFailuresDoNotPanicAndKeepOldFiltering(t *testing.T) { + t.Setenv("XDG_CURRENT_DESKTOP", "DDE") + ls := testLanguageSupport(t, `tr::firefox:firefox-locale- +wa:de::hunspell-de-at-frami +`) + + // dpkg fails: no trigger counts as installed, so only the empty trigger + // contributes, and the failing command is not retried per package + fake := &fakeCommands{ + dpkgErr: assert.AnError, + aptOut: `firefox-locale-de: + Installed: (none) + Candidate: 1.0 +hunspell-de-at-frami: + Installed: (none) + Candidate: 1.0 +`, + } + fake.install(t) + assert.ElementsMatch(t, []string{"hunspell-de-at-frami"}, ls.ByLocale("de_DE", false)) + assert.Len(t, fake.callsOf(dpkgCmdName), 1) + + // apt fails: nothing is installable + fake.calls = nil + fake.dpkgErr = nil + fake.aptErr = assert.AnError + assert.Empty(t, ls.ByLocale("de_DE", false)) + assert.Len(t, fake.callsOf(aptCmdName), 1) +} + +func TestLanguageSupportRefreshesPackageState(t *testing.T) { + ls := testLanguageSupport(t, "fn:ja::fonts-example\n") + fake := &fakeCommands{aptOut: "fonts-example:\n Candidate: 1.0\n"} + fake.install(t) + assert.ElementsMatch(t, []string{"fonts-example"}, ls.ByLocale("ja_JP", false)) + + // Reusing the object after installation must not return stale missing packages. + fake.dpkgOut = "fonts-example installed\n" + assert.Empty(t, ls.ByLocale("ja_JP", false)) + + // Repository changes must also be visible to the next query. + fake.aptOut = "fonts-example:\n Candidate: (none)\n" + assert.Empty(t, ls.ByLocale("ja_JP", true)) +}