From 6f3b644613c75df9c5b48a231392bccfbba45be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentino=20Me=C4=91imorec?= Date: Sun, 18 Oct 2026 14:51:42 +0200 Subject: [PATCH 1/2] validate: backup exclude among the keys that landed The control plane echoes vallic.yaml's new `backup.exclude` as `backup_exclude`: paths to leave out of the files backup, beside the framework's own caches. Printed after extras, in the order the control plane describes them in. Without it, a file declaring one would read as not having been read, which is what a missing line here means. Co-Authored-By: Claude Opus 5.5 --- internal/api/models.go | 4 ++++ internal/cli/validate.go | 1 + internal/cli/validate_test.go | 2 ++ 3 files changed, 7 insertions(+) diff --git a/internal/api/models.go b/internal/api/models.go index c9aa9b3..1431f4b 100644 --- a/internal/api/models.go +++ b/internal/api/models.go @@ -1052,6 +1052,10 @@ type Manifest struct { // platform did not write. Declared in the manifest, defined by a compose // fragment the control plane does not echo and this does not ask for. Extras []ManifestExtra `json:"extras"` + + // BackupExclude is what the files backup leaves out beside the + // framework's own caches, as the file wrote it. + BackupExclude []string `json:"backup_exclude"` } // ManifestService is one service a repository asked to run beside it. diff --git a/internal/cli/validate.go b/internal/cli/validate.go index f16063b..d3af64a 100644 --- a/internal/cli/validate.go +++ b/internal/cli/validate.go @@ -330,6 +330,7 @@ func manifestEntries(manifest *api.Manifest) []manifestEntry { add("required env", strings.Join(manifest.RequiredEnv, ", ")) add("mounts", strings.Join(manifest.Mounts, ", ")) add("extras", manifestExtras(manifest.Extras)) + add("backup exclude", strings.Join(manifest.BackupExclude, ", ")) return entries } diff --git a/internal/cli/validate_test.go b/internal/cli/validate_test.go index 3d87863..2615a28 100644 --- a/internal/cli/validate_test.go +++ b/internal/cli/validate_test.go @@ -108,6 +108,7 @@ func TestManifestEntriesLeaveOutWhatWasNotRead(t *testing.T) { HealthTimeout: 30, RequiredEnv: []string{"STRIPE_KEY"}, Mounts: []string{"public/files"}, + BackupExclude: []string{"web/sites/default/files/backup_migrate", "private/exports/*.csv"}, }, want: []manifestEntry{ {Key: "type", Value: "drupal"}, @@ -122,6 +123,7 @@ func TestManifestEntriesLeaveOutWhatWasNotRead(t *testing.T) { {Key: "health", Value: "/up, 30s"}, {Key: "required env", Value: "STRIPE_KEY"}, {Key: "mounts", Value: "public/files"}, + {Key: "backup exclude", Value: "web/sites/default/files/backup_migrate, private/exports/*.csv"}, }, }, { From 4af0d0ad22b74193bf73bd4f94ec1bb71cac888b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentino=20Me=C4=91imorec?= Date: Sun, 18 Oct 2026 21:36:52 +0200 Subject: [PATCH 2/2] Improver console db handling. Added new endpoints --- README.md | 22 +- USAGE.md | 101 ++++- internal/api/models.go | 174 ++++++++ internal/api/routes.go | 208 ++++++++++ internal/auth/device.go | 10 +- internal/cli/command.go | 66 ++++ internal/cli/deploy.go | 50 +++ internal/cli/dump.go | 658 +++++++++++++++++++++++++++++++ internal/cli/dump_test.go | 276 +++++++++++++ internal/cli/environment.go | 1 + internal/cli/init.go | 555 +++++++++++++++++++++++++- internal/cli/init_test.go | 309 ++++++++++++++- internal/cli/login.go | 11 +- internal/cli/operate.go | 361 +++++++++++++++++ internal/cli/operate_test.go | 105 +++++ internal/cli/people.go | 442 +++++++++++++++++++++ internal/cli/root.go | 7 + internal/cli/shell.go | 186 +++++++-- internal/cli/sshkey.go | 282 +++++++++++++ internal/cli/sshkey_test.go | 71 ++++ internal/cli/team.go | 7 +- internal/cli/variable.go | 192 +++++++++ internal/resolve/resolve.go | 2 +- internal/resolve/resolve_test.go | 8 +- internal/ssh/ssh.go | 16 + internal/ssh/ssh_test.go | 25 ++ 26 files changed, 4068 insertions(+), 77 deletions(-) create mode 100644 internal/cli/dump.go create mode 100644 internal/cli/dump_test.go create mode 100644 internal/cli/operate.go create mode 100644 internal/cli/operate_test.go create mode 100644 internal/cli/people.go create mode 100644 internal/cli/sshkey.go create mode 100644 internal/cli/sshkey_test.go diff --git a/README.md b/README.md index 4fa8d0b..0f1a2b8 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,9 @@ Switching current -> releases/412 $ vallic ssh vallic@acme-webshop-16-production:/var/www/html$ drush status -$ vallic db export > dump.sql -$ gunzip -c dump.sql.gz | vallic db import staging +$ vallic db dump --gzip +✓ Wrote acme--production--dump.sql.gz (88.4 MB) +$ vallic db import staging -f acme--production--dump.sql.gz $ vallic mount download ./files $ vallic tunnel staging db @@ -216,23 +217,28 @@ when it is left off. | | | |---|---| -| `login` · `logout` · `whoami` · `team` | Signing in, and which team a credential reaches | +| `login` · `logout` · `whoami` · `team list\|switch` | Signing in, and which team a credential reaches | +| `team people\|invite\|role\|remove\|withdraw` | Who is in the team, inviting more, their roles | +| `ssh-key list\|add\|delete` | The SSH keys on your account, which every shell and dump logs in with | | `project` · `env` · `server` · `status` | What exists | | `url [--all] [--open]` | The browsable address, alone on stdout. `--all` lists every name the edge routes | | `deploy` · `rollback` · `redeploy` · `release list` | Shipping | -| `activity list` · `activity get` · `activity log` | What the platform has been doing, and the log of one task | +| `activity list` · `activity get` · `activity log` · `activity cancel` | What the platform has been doing, the log of one task, and calling off one that has not started | | `logs` | Where the site's own request and error logs go — the project's destination and the machine's copy. Not a tail: Vallic Cloud does not keep them | -| `ssh` · `sql` · `db` · `mount` | Getting inside | +| `ssh` · `sql` · `db dump\|export\|import\|size` · `mount` | Getting inside. `db dump` writes a file named for the project and environment, all of it or `--table`s; `db size` shows the largest tables; `sql -- ` runs one query | | `drush` | `drush` in the container, over the same forced command `ssh --` uses | -| `var list\|get\|set\|delete\|apply` | Variables, at either scope; `apply` restarts the site with them now | +| `var list\|get\|set\|delete\|import\|apply` | Variables, at either scope; `import` adds a `.env`, `apply` restarts the site with them now | | `domain list\|add\|verify\|delete` | Hostnames, and the DNS each one needs | | `service list` | The stack, and the versions that go in `vallic.yaml` | | `backup list\|create\|download\|restore` | Copies, and putting one back | -| `env create` · `env source` · `env delete` | Raising, repointing and removing an environment | +| `env create` · `env source` · `env maintenance` · `env delete` | Raising, repointing, taking offline and removing an environment | +| `cdn show\|clear` · `metrics` | What the CDN caches, and emptying it; CPU, memory and disk over a window | | `validate [--local]` | Checking `vallic.yaml` before you commit it | -| `init` | Writes a starting `vallic.yaml` from what the checkout looks like | +| `init` | Writes a starting `vallic.yaml` from what the checkout looks like: Drupal, Laravel, WordPress, PHP, Node.js (or Bun), Go, Rust, Python, Django, Ruby and Rails | | `link` | Records which project this checkout is, when the git remote cannot say | +Upsun's and Platform.sh's spelling works too: `vallic db:dump`, `vallic environment:list` and `vallic user:add` are `db dump`, `env list` and `team invite`. + Four of these are worth knowing about before you need them. **A secret is never readable, by anybody.** Not masked for a viewer and shown diff --git a/USAGE.md b/USAGE.md index b65d5af..85f31e5 100644 --- a/USAGE.md +++ b/USAGE.md @@ -20,7 +20,13 @@ vallic login --token # Who the stored credential belongs to, and which teams it reaches vallic whoami -vallic team +vallic team list + +# Put your SSH key on your account, for ssh, db and mount: the first of +# ~/.ssh/id_ed25519.pub, id_ecdsa.pub and id_rsa.pub, or the one you name +vallic ssh-key add +vallic ssh-key add ~/.ssh/vallic.pub --label work-laptop +vallic ssh-key list # Record a project for directories that are not a clone of its repository. # One for the whole machine: a clone's own remote still answers first. @@ -54,8 +60,10 @@ vallic url staging --open ## vallic.yaml ```bash -# A starting file from what the checkout looks like -vallic init --type drupal +# A starting file from what the checkout looks like: composer.json, go.mod, +# Cargo.toml, pyproject.toml, Gemfile or package.json (on Bun with a bun.lock) +vallic init +vallic init --type rust # Check it before committing: on its own, or against what an environment runs vallic validate --local @@ -72,10 +80,30 @@ vallic env create feature/checkout --type development --auto-deploy vallic env source staging --branch release/2.0 vallic env source staging --auto-deploy false +# Put staging behind its offline page while something is fixed, and back +vallic env maintenance on staging --note "Back at 15:00" +vallic env maintenance off staging + # Remove an environment and its machines vallic env delete feature-checkout ``` +## The team + +```bash +# Who is in the team, and the invitations still waiting +vallic team people + +# Invite somebody; the link is printed once and no mail is sent +vallic team invite ana@example.com --role developer +vallic team invite contractor@example.com --role developer --projects acme-site + +# Another role, take somebody off, or take an invitation back +vallic team role ana@example.com admin +vallic team remove contractor@example.com +vallic team withdraw old@example.com +``` + ## Building and deploying ```bash @@ -106,8 +134,15 @@ vallic activity list production --type deploy,build --limit 20 vallic activity get 1234 vallic activity log 1234 --follow +# Call off a deploy still waiting in the queue; one already running finishes +vallic activity cancel 1235 + # Where the site's logs go vallic logs production + +# CPU, memory and disk: now, on average and at peak +vallic metrics production +vallic metrics production --window 7d ``` ## A shell and commands @@ -179,16 +214,36 @@ vallic mount upload staging ./files --delete ## Databases ```bash -# A copy of production's database, then onto staging +# A copy of production's database, in a file named for it, then onto staging +vallic db dump production +vallic db import staging --file acme--production--dump.sql + +# Compressed, with the time in the name so the last one is kept; import +# decompresses whatever it is given +vallic db dump production --gzip --timestamp +vallic db import staging -f acme--production--20261018-031500-UTC--dump.sql.gz + +# Somewhere else, or named outright +vallic db dump production --directory ~/dumps +vallic db dump production --file before-upgrade.sql.gz + +# Part of it: some tables, all but some, or the schema alone +vallic db dump production --table users --table orders +vallic db dump production --exclude-table cache_render,sessions +vallic db dump production --schema-only + +# How much it holds, and the largest tables +vallic db size production + +# A stream, for a pipeline vallic db export production > production.sql +vallic db export production --gzip > production.sql.gz vallic db import staging < production.sql -# Compressed on the way out -vallic db export production | gzip > production.sql.gz -gunzip -c production.sql.gz | vallic db import staging - -# An interactive prompt +# An interactive prompt, one query, or a file of them vallic sql staging +vallic sql production -- 'SELECT COUNT(*) FROM users' +vallic sql staging < fix.sql # A local port onto the database, for a desktop client, and the credentials vallic tunnel staging db --port 3307 @@ -226,6 +281,11 @@ vallic var set PAYMENT_KEY sk_live_example production --secret vallic var get API_URL staging vallic var delete OLD_FLAG staging +# Add every variable in a .env, all or none; see first what it would add +vallic var import .env.production production --dry-run +vallic var import .env.production production +vallic var import .env.shared --scope project --secret STRIPE_KEY,DB_PASSWORD + # Restart the site so changed variables take effect now vallic var apply production ``` @@ -247,6 +307,29 @@ vallic domain delete old.example.com production `domain verify` exits 0 when the answer is "not yet", so a retry loop does not fail on a record that has not propagated. +## The CDN + +```bash +# What it caches, and for how long +vallic cdn show production + +# Empty it after a change the site's own cache headers do not cover +vallic cdn clear production +``` + +## Coming from Upsun + +Upsun's `noun:verb` spelling is read as the two words it stands for, and the +commands spelled differently here are mapped to what does the same thing: + +```bash +vallic db:dump production # vallic db dump production +vallic environment:list # vallic env list +vallic db:sql production # vallic sql production +vallic user:add ana@example.com --role developer # vallic team invite … +vallic self:update # vallic self-update +``` + ## In scripts and CI ```bash diff --git a/internal/api/models.go b/internal/api/models.go index 1431f4b..4316afa 100644 --- a/internal/api/models.go +++ b/internal/api/models.go @@ -188,6 +188,14 @@ type SSHTarget struct { DBImport string `json:"db_import"` DBCLI string `json:"db_cli"` + // DBSize is the verb that reports the database's size, and + // DBExportOptions what DBExport takes after its name: `--table` and the + // rest. Empty where the machines' agent is older than the shell that + // reads them, and against a control plane that predates both, so a client + // asks for neither where it would be refused. + DBSize string `json:"db_size"` + DBExportOptions []string `json:"db_export_options"` + // FilesPath is the one writable directory that is the same for every // project type and survives every deploy. What rsync should be aimed at. FilesPath string `json:"files_path"` @@ -1381,3 +1389,169 @@ type LogDestination struct { KindLabel string `json:"kind_label"` Name string `json:"name"` } + +// TaskCancelled is the answer to a cancel. +type TaskCancelled struct { + Task struct { + ID int `json:"id"` + State string `json:"state"` + } `json:"task"` + + Message string `json:"message"` +} + +// SSHKey is one public key on a person's account. Never the key itself: the +// fingerprint is what somebody compares against `ssh-keygen -lf`. +type SSHKey struct { + ID int `json:"id"` + Label string `json:"label"` + Type string `json:"type"` + Bits *int `json:"bits"` + Fingerprint string `json:"fingerprint"` + Comment string `json:"comment"` + + // LastUsed is an ISO 8601 time, or nil for a key never used to log in. + LastUsed *string `json:"last_used"` +} + +// SSHKeyAdded is the answer to adding a key, with the control plane's own +// sentence naming its fingerprint. +type SSHKeyAdded struct { + Key SSHKey `json:"ssh_key"` + Message string `json:"message"` +} + +// CDN is an environment's CDN settings, as the edge has them now. +// +// Read from the pull zone rather than from a copy here, so this is what the +// edge is doing. Only the values: the options and limits beside them are the +// console's form's, and a client that printed them would be printing a form. +type CDN struct { + Values struct { + Dynamic bool `json:"dynamic"` + + // The browser cache times in seconds, or nil where the site's own + // headers decide. + BrowserCachePages *int `json:"browser_cache_pages"` + BrowserCacheFiles *int `json:"browser_cache_files"` + + Vary []string `json:"vary"` + Cookies []string `json:"cookies"` + Headers []string `json:"headers"` + Blocked []string `json:"blocked"` + SkipPaths []string `json:"skip_paths"` + SessionCookies []string `json:"session_cookies"` + } `json:"values"` +} + +// Said is an answer that is a sentence and nothing else worth reading. +type Said struct { + Message string `json:"message"` +} + +// Maintenance is whether an environment is behind its offline page. +type Maintenance struct { + Maintenance bool `json:"maintenance"` + Note string `json:"note"` + Message string `json:"message"` +} + +// ParsedVariable is one line of a .env, as the control plane read it. +type ParsedVariable struct { + Name string `json:"name"` + Value string `json:"value"` + + // Secret is the control plane's guess from the name, which an import + // follows unless told which ones are. + Secret bool `json:"secret"` +} + +// ParsedVariables is what a .env would import, before it does. +type ParsedVariables struct { + Variables []ParsedVariable `json:"variables"` + Summary string `json:"summary"` +} + +// VariablesImported is the answer to an import, which adds all or none. +type VariablesImported struct { + Imported int `json:"imported"` + TakesEffect string `json:"takes_effect"` + Message string `json:"message"` +} + +// TeamPeople is a team's roster. +// +// Invitations and Roles are empty for somebody who may not invite: a viewer +// sees who is in the team and nothing about who is about to join it. +type TeamPeople struct { + Members []TeamMember `json:"members"` + Invitations []TeamInvitation `json:"invitations"` + Roles []struct { + Value string `json:"value"` + Label string `json:"label"` + Description string `json:"description"` + Scoped bool `json:"scoped"` + } `json:"roles"` + Projects []struct { + ID int `json:"id"` + Label string `json:"label"` + } `json:"projects"` +} + +// TeamMember is one membership. +type TeamMember struct { + ID int `json:"id"` + User struct { + ID int `json:"id"` + Name string `json:"name"` + Email *string `json:"email"` + } `json:"user"` + Role string `json:"role"` + + // Projects are the ids the role is narrowed to; empty is every project. + Projects []int `json:"projects"` + IsSelf bool `json:"is_self"` +} + +// TeamInvitation is an invitation still waiting, without its link. +type TeamInvitation struct { + ID int `json:"id"` + Email string `json:"email"` + Role string `json:"role"` + Projects []int `json:"projects"` + State string `json:"state"` + Created int64 `json:"created"` + Expires int64 `json:"expires"` +} + +// Invited is a new invitation, with the one copy of its link there will ever +// be: only its digest is kept, and no mail is sent. +type Invited struct { + Invitation TeamInvitation `json:"invitation"` + Link string `json:"link"` +} + +// MembershipChanged is the answer to a change of role. +type MembershipChanged struct { + Membership TeamMember `json:"membership"` + Message string `json:"message"` +} + +// Resources is an environment's CPU, memory and disk over a window. +type Resources struct { + Environment string `json:"environment"` + Window string `json:"window"` + Step int `json:"step"` + Points []ResourcePoint `json:"points"` +} + +// ResourcePoint is one step's average. +type ResourcePoint struct { + Taken int64 `json:"taken"` + CPUPercent float64 `json:"cpu_percent"` + MemoryBytes float64 `json:"memory_bytes"` + + // DiskBytes is nil for a step with no disk reading, which is a gap and + // not an empty disk. + DiskBytes *float64 `json:"disk_bytes"` +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 581aa2f..849f919 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -667,3 +667,211 @@ func (c *Client) LatestCLI(ctx context.Context, channel string) (*CLIRelease, er return &out, nil } + +// CancelTask cancels a task that has not started. +// +// Only one that is still waiting: a deploy half way through is not something +// a cancel can put back, so the control plane answers 409 for anything a +// machine has already claimed, in its own words. +func (c *Client) CancelTask(ctx context.Context, id int) (*TaskCancelled, error) { + var out TaskCancelled + + if err := c.post(ctx, fmt.Sprintf("/tasks/%d/cancel", id), nil, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// SSHKeys lists the public keys on the signed-in person's account. +// +// The person's own, whichever team the credential is for: a key opens every +// environment the person may open. Not reachable with a personal access token, +// which the control plane refuses for everything under /me but /me itself. +func (c *Client) SSHKeys(ctx context.Context) ([]SSHKey, error) { + var out struct { + Keys []SSHKey `json:"ssh_keys"` + } + + if err := c.get(ctx, "/me/ssh-keys", &out); err != nil { + return nil, err + } + + return out.Keys, nil +} + +// AddSSHKey puts a public key on the signed-in person's account. +func (c *Client) AddSSHKey(ctx context.Context, label, publicKey string) (*SSHKeyAdded, error) { + var out SSHKeyAdded + + body := map[string]string{"label": label, "public_key": publicKey} + + if err := c.post(ctx, "/me/ssh-keys", body, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// RemoveSSHKey takes a key off the signed-in person's account. The control +// plane answers with no body. +func (c *Client) RemoveSSHKey(ctx context.Context, id int) error { + return c.delete(ctx, fmt.Sprintf("/me/ssh-keys/%d", id), nil) +} + +// CDN reads an environment's CDN settings from the edge. 404 where the +// environment is not served through the CDN, 409 where the CDN cannot be +// asked just now; both in the control plane's words. +func (c *Client) CDN(ctx context.Context, environmentID int) (*CDN, error) { + var out CDN + + if err := c.get(ctx, fmt.Sprintf("/environments/%d/cdn", environmentID), &out); err != nil { + return nil, err + } + + return &out, nil +} + +// ClearCDN empties the CDN's cache for an environment. Accepted rather than +// done: the edge serves fresh copies within a minute or so. +func (c *Client) ClearCDN(ctx context.Context, environmentID int) (*Said, error) { + var out Said + + if err := c.post(ctx, fmt.Sprintf("/environments/%d/cdn/clear", environmentID), nil, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// SetMaintenance puts an environment behind its offline page, or takes it +// out. note is the line the page shows; nil keeps the one it has. +func (c *Client) SetMaintenance(ctx context.Context, environmentID int, on bool, note *string) (*Maintenance, error) { + var out Maintenance + + body := map[string]any{"maintenance": on} + if note != nil { + body["note"] = *note + } + + if err := c.post(ctx, fmt.Sprintf("/environments/%d/maintenance", environmentID), body, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// ParseVariables reads a .env the way an import would, and touches nothing. +func (c *Client) ParseVariables(ctx context.Context, pasted string) (*ParsedVariables, error) { + var out ParsedVariables + + if err := c.post(ctx, "/variables/parse", map[string]any{"pasted": pasted}, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// ImportEnvironmentVariables adds every variable in a .env to an environment, +// all or none. secret names exactly the ones to keep secret; nil leaves that +// to the control plane, which marks the names that read like credentials. +func (c *Client) ImportEnvironmentVariables(ctx context.Context, environmentID int, pasted string, secret []string) (*VariablesImported, error) { + return c.importVariables(ctx, fmt.Sprintf("/environments/%d/variables/import", environmentID), pasted, secret) +} + +// ImportProjectVariables is the same, for the project's own. +func (c *Client) ImportProjectVariables(ctx context.Context, projectID int, pasted string, secret []string) (*VariablesImported, error) { + return c.importVariables(ctx, fmt.Sprintf("/projects/%d/variables/import", projectID), pasted, secret) +} + +func (c *Client) importVariables(ctx context.Context, path, pasted string, secret []string) (*VariablesImported, error) { + var out VariablesImported + + body := map[string]any{"pasted": pasted} + if secret != nil { + body["secret"] = secret + } + + if err := c.post(ctx, path, body, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// TeamPeople reads a team's members and the invitations still waiting. +func (c *Client) TeamPeople(ctx context.Context, teamID int) (*TeamPeople, error) { + var out TeamPeople + + if err := c.get(ctx, fmt.Sprintf("/teams/%d/people", teamID), &out); err != nil { + return nil, err + } + + return &out, nil +} + +// Invite asks somebody to join a team. projects narrows the role to those +// projects; nil is every project. +func (c *Client) Invite(ctx context.Context, teamID int, email, role string, projects []int) (*Invited, error) { + var out Invited + + body := map[string]any{"email": email, "role": role} + if projects != nil { + body["projects"] = projects + } else { + body["scope"] = "all" + } + + if err := c.post(ctx, fmt.Sprintf("/teams/%d/invitations", teamID), body, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// WithdrawInvitation takes an invitation back; its link stops working. +func (c *Client) WithdrawInvitation(ctx context.Context, invitationID int) error { + return c.delete(ctx, fmt.Sprintf("/invitations/%d", invitationID), nil) +} + +// ChangeMembership gives a member another role. projects as for Invite. +func (c *Client) ChangeMembership(ctx context.Context, membershipID int, role string, projects []int) (*MembershipChanged, error) { + var out MembershipChanged + + fields := map[string]any{"role": role} + if projects != nil { + fields["projects"] = projects + } else { + fields["scope"] = "all" + } + + body, err := json.Marshal(fields) + if err != nil { + return nil, err + } + + if err := c.do(ctx, http.MethodPatch, fmt.Sprintf("/memberships/%d", membershipID), body, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// RemoveMembership takes somebody off a team, or, for one's own, leaves it. +func (c *Client) RemoveMembership(ctx context.Context, membershipID int) error { + return c.delete(ctx, fmt.Sprintf("/memberships/%d", membershipID), nil) +} + +// Resources reads an environment's CPU, memory and disk over a window: +// 1h, 24h, 7d, 30d or 90d. +func (c *Client) Resources(ctx context.Context, environmentID int, window string) (*Resources, error) { + var out Resources + + path := fmt.Sprintf("/environments/%d/resources?window=%s", environmentID, url.QueryEscape(window)) + + if err := c.get(ctx, path, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/internal/auth/device.go b/internal/auth/device.go index b8b6c31..d299f19 100644 --- a/internal/auth/device.go +++ b/internal/auth/device.go @@ -44,11 +44,11 @@ type DeviceAuth struct { DeviceCode string `json:"device_code"` UserCode string `json:"user_code"` - // VerificationURI is the page to open. VerificationURIComplete has the - // code already in it, and is what the CLI should offer first — a code - // typed by hand is a code mistyped by hand. - VerificationURI string `json:"verification_uri"` - VerificationURIComplete string `json:"verification_uri_complete,omitempty"` + // VerificationURI is the page to open, the same for everybody: the code + // is typed there. A verification_uri_complete, with the code in it, is + // not read — a link that fills the code in is what somebody would send a + // person to approve their sign-in instead. + VerificationURI string `json:"verification_uri"` ExpiresIn int `json:"expires_in"` Interval int `json:"interval,omitempty"` diff --git a/internal/cli/command.go b/internal/cli/command.go index ebac299..ff58610 100644 --- a/internal/cli/command.go +++ b/internal/cli/command.go @@ -15,6 +15,7 @@ import ( "flag" "fmt" "io" + "slices" "sort" "strings" "time" @@ -111,6 +112,15 @@ func Execute(ctx context.Context, root *Command, env *Env, args []string) error break } + // Upsun's and Platform.sh's spelling, `db:dump`, read as the two + // words it stands for. At the top only, where theirs start, and only + // for a word no command here is called. + if len(path) == 1 && strings.Contains(args[0], ":") && cmd.lookup(args[0]) == nil { + if words, ok := colonSpelling(cmd, args[0]); ok { + args = append(words, args[1:]...) + } + } + child := cmd.lookup(args[0]) if child == nil { // Named rather than silently treated as an argument: a typo in a @@ -201,6 +211,62 @@ func Execute(ctx context.Context, root *Command, env *Env, args []string) error return err } +// upsunNames are Upsun's commands that are spelled differently here, beyond +// the colon: a prompt is `sql` rather than `db:sql`, and a team's people are +// the team's rather than a project's users. Each maps to what does the same +// thing; a command with no equivalent is not listed, and is refused by name +// like any other. +var upsunNames = map[string][]string{ + "auth:browser-login": {"login"}, + "auth:api-token-login": {"login", "--token"}, + "auth:info": {"whoami"}, + "auth:logout": {"logout"}, + "backup:get": {"backup", "download"}, + "db:sql": {"sql"}, + "environment:deploy": {"deploy"}, + "environment:drush": {"drush"}, + "environment:logs": {"logs"}, + "environment:redeploy": {"redeploy"}, + "environment:ssh": {"ssh"}, + "environment:url": {"url"}, + "metrics:all": {"metrics"}, + "project:set-remote": {"link"}, + "route:list": {"url", "--all"}, + "tunnel:open": {"tunnel"}, + "tunnel:single": {"tunnel"}, + "user:add": {"team", "invite"}, + "user:delete": {"team", "remove"}, + "user:list": {"team", "people"}, + "user:update": {"team", "role"}, + "variable:create": {"var", "set"}, + "variable:update": {"var", "set"}, +} + +// colonSpelling reads `noun:verb` as the words it stands for here. +// +// The table first, for the names that differ; then the colon as a space, +// which is most of them — `environment:list`, `mount:upload`, +// `backup:restore` — since the nouns are the same entities; then as a +// hyphen, for `self:update`. Nothing matching is left as it was, so the +// refusal names what was typed. +func colonSpelling(root *Command, word string) ([]string, bool) { + if words, ok := upsunNames[strings.ToLower(word)]; ok { + return slices.Clone(words), true + } + + parts := strings.Split(word, ":") + + if root.lookup(parts[0]) != nil { + return parts, true + } + + if joined := strings.Join(parts, "-"); root.lookup(joined) != nil { + return []string{joined}, true + } + + return nil, false +} + // splitPassthrough separates the arguments at the first bare "--". // // found distinguishes "no separator" from "a separator with nothing after diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index 12ad4ab..fde017a 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -513,6 +513,7 @@ which go to wherever the project sends them and never to the control plane.`, Children: []*Command{ activityListCommand(), activityLogCommand(), + activityCancelCommand(), { Name: "get", Summary: "show one task", @@ -568,6 +569,55 @@ which go to wherever the project sends them and never to the control plane.`, } } +// activityCancelCommand cancels a task before it starts. +// +// Upsun's activity:cancel, with the platform's narrower answer: only work no +// machine has claimed yet can be called off. A deploy half way through has +// already changed something, and stopping it would leave that half in place, +// so the control plane refuses and says it had started. +func activityCancelCommand() *Command { + return &Command{ + Name: "cancel", + Summary: "call off a task that has not started", + Usage: "activity cancel ", + Long: `Cancels queued work — a deploy waiting behind another, a backup not yet +picked up — by the id ` + "`vallic activity list`" + ` prints. Work a machine has +already started is not stopped: it runs to the end, and this says so.`, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return Usagef(nil, "name a task id, as `vallic activity list` shows it") + } + + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + id, err := atoi(args[0]) + if err != nil { + return Usagef(nil, "%q is not a task id", args[0]) + } + + client, err := env.Client() + if err != nil { + return err + } + + cancelled, err := client.CancelTask(ctx, id) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"task": cancelled.Task}) + } + + env.Printer.Good("%s", firstNonEmpty(cancelled.Message, fmt.Sprintf("Task #%d is cancelled.", id))) + + return nil + }, + } +} + // activityLogCommand prints a task's output. func activityLogCommand() *Command { var ( diff --git a/internal/cli/dump.go b/internal/cli/dump.go new file mode 100644 index 0000000..83d6852 --- /dev/null +++ b/internal/cli/dump.go @@ -0,0 +1,658 @@ +package cli + +import ( + "bufio" + "compress/gzip" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/vallic/vallic-cli/internal/output" + vssh "github.com/vallic/vallic-cli/internal/ssh" + "github.com/vallic/vallic-cli/internal/terminal" +) + +// dbDumpCommand writes a database dump to a file it names itself. +// +// Upsun's `db:dump`, which is the command people coming from it type first: +// the file is named after the project and the environment, so a dump needs +// nothing but the environment, and two dumps of different environments never +// land on the same name. `db export` is still the stream, for a pipeline. +func dbDumpCommand() *Command { + var ( + identity string + dryRun bool + file string + directory string + gzipped bool + stamped bool + stdout bool + part dumpPart + ) + + return &Command{ + Name: "dump", + Summary: "write a dump of the database to a file named for it", + Usage: "db dump [] [--file ] [--directory ] [--gzip] [--timestamp] [--stdout] [--table ]... [--exclude-table ]... [--schema-only]", + Long: `Writes the database to a file in the working directory, named after the +project and the environment, and says where: + + vallic db dump production + ✓ Wrote acme--production--dump.sql (412 MB) + +--gzip (-z) compresses it and names it .gz, and a --file ending in .gz is +compressed without being asked. --timestamp (-t) puts the time of the dump in +the name, in UTC, so a nightly dump never replaces the last one. --directory +(-d) puts the named file somewhere else, and --file (-f) names it outright. + +The file appears only once the dump is complete: it is written beside its +name and moved there at the end, so a dump that fails half way leaves nothing +that looks like a good one. A file already there is replaced only when you +confirm it. + +--stdout (-o) writes to standard output instead, which is what db export does. + +Part of the database, as Upsun's db:dump takes it: + + vallic db dump production --table users --table orders + vallic db dump production --exclude-table cache_render,sessions + vallic db dump production --schema-only + +Each flag takes a table, or several separated by commas, and can be repeated; +a table in another PostgreSQL schema is schema.table. The name the file gets +says which part it holds. Names are exact: there are no wildcards, because +mysqldump has none. + +What runs on the far side is the same db-export verb: the engine's own client +in the container, with credentials that never reach this machine.`, + Flags: func(fs *flag.FlagSet) { + identityFlag(fs, &identity) + fs.BoolVar(&dryRun, "dry-run", false, "print the command instead of running it") + fs.StringVar(&file, "file", "", "the file to write, instead of the name chosen for it") + fs.StringVar(&file, "f", "", "the file to write") + fs.StringVar(&directory, "directory", "", "the directory the named file goes in") + fs.StringVar(&directory, "d", "", "the directory the named file goes in") + fs.BoolVar(&gzipped, "gzip", false, "compress the dump") + fs.BoolVar(&gzipped, "z", false, "compress the dump") + fs.BoolVar(&stamped, "timestamp", false, "put the time of the dump in its name") + fs.BoolVar(&stamped, "t", false, "put the time of the dump in its name") + fs.BoolVar(&stdout, "stdout", false, "write to standard output instead of a file") + fs.BoolVar(&stdout, "o", false, "write to standard output instead of a file") + part.flags(fs) + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + if stdout && (file != "" || directory != "" || stamped) { + return Usagef(nil, "--stdout writes no file, so it takes no --file, --directory or --timestamp") + } + + if file != "" && directory != "" { + return Usagef(nil, "--file names the whole path; --directory is for the name chosen when there is no --file") + } + + if err := part.check(); err != nil { + return err + } + + target, resolved, err := env.sshTargetFor(ctx, first(args), identity) + if err != nil { + return err + } + + verb, err := part.verb(target) + if err != nil { + return err + } + + target.Compress = true + process := target.Verb(verb, false) + + if stdout { + return streamDump(env, process, gzipped, dryRun, target.Host) + } + + path := file + if path == "" { + path = filepath.Join(directory, dumpFilename(resolved, part, gzipped, stamped, time.Now())) + } + + if strings.HasSuffix(path, ".gz") { + gzipped = true + } + + if dryRun { + if gzipped { + env.Printer.Line("%s | gzip > %s", vssh.Describe(process), vssh.Quote(path)) + } else { + env.Printer.Line("%s > %s", vssh.Describe(process), vssh.Quote(path)) + } + + return nil + } + + return writeDump(env, process, path, gzipped, resolved.Named()) + }, + } +} + +// dumpFilename is the name a dump is written under when none is given. +// +// Upsun's shape, `----dump.sql`, with the double dash +// that keeps a hyphenated project apart from its environment, and the part of +// the database after them as Upsun names it: the tables, `excl-` and the +// tables left out, `schema`. The time, when asked for, is UTC and to the +// second, so names sort in the order the dumps were taken. +func dumpFilename(target *Target, part dumpPart, gzipped, stamped bool, now time.Time) string { + var parts []string + + if target != nil && target.Project != nil && target.Project.MachineName != "" { + parts = append(parts, target.Project.MachineName) + } + + if target != nil && target.Environment != nil { + parts = append(parts, firstNonEmpty(target.Environment.Name, target.Environment.Slug)) + } + + if len(part.tables) > 0 { + parts = append(parts, strings.Join(part.tables, ",")) + } + + if len(part.excluded) > 0 { + parts = append(parts, "excl-"+strings.Join(part.excluded, ",")) + } + + if part.schemaOnly { + parts = append(parts, "schema") + } + + if stamped { + parts = append(parts, now.UTC().Format("20060102-150405")+"-UTC") + } + + name := strings.Join(append(parts, "dump.sql"), "--") + + if gzipped { + name += ".gz" + } + + // A name comes from the platform, and a slash in it would be a directory + // nobody asked for. + return strings.Map(func(r rune) rune { + if r == '/' || r == '\\' || r == 0 { + return '-' + } + + return r + }, name) +} + +// dumpPart is which part of the database a dump takes: every table unless +// told otherwise, and their rows unless --schema-only. +type dumpPart struct { + tables tableList + excluded tableList + schemaOnly bool +} + +func (p *dumpPart) flags(fs *flag.FlagSet) { + fs.Var(&p.tables, "table", "dump only this table; repeat it, or separate several with commas") + fs.Var(&p.excluded, "exclude-table", "leave this table out; repeat it, or separate several with commas") + fs.BoolVar(&p.schemaOnly, "schema-only", false, "the tables' definitions without their rows") +} + +// The names db-export takes, as the agent checks them: letters, digits, `_` +// and `-`, with one `.` before a table in another schema. Checked here too so +// a typo is said before anything connects. +var dumpTableName = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_-]*(\.[A-Za-z0-9_][A-Za-z0-9_-]*)?$`) + +func (p *dumpPart) check() error { + for _, table := range append(slices.Clone(p.tables), p.excluded...) { + if !dumpTableName.MatchString(table) { + return Usagef(nil, "%q is not a table name a dump takes: letters, digits, _ and -, and schema.table for another schema; there are no wildcards", table) + } + } + + for _, table := range p.tables { + if slices.Contains(p.excluded, table) { + return Usagef(nil, "%s is both the table to dump and one to leave out", table) + } + } + + return nil +} + +// verb is the export to ask for: the control plane's verb, and the options +// after it where the environment takes them. +// +// What it takes is the control plane's to say. A machine whose agent reads no +// options would run `db-export --table=users` as a command and answer "not +// found", so an option it does not list is refused here, in a sentence. +func (p *dumpPart) verb(target *vssh.Target) (string, error) { + if target.DBExport == "" { + return "", fmt.Errorf("this environment runs no database") + } + + words := []string{target.DBExport} + + ask := func(option string, values ...string) error { + if !slices.Contains(target.DBExportOptions, option) { + return fmt.Errorf( + "this environment does not offer %s yet: its machines' agent, or the control plane, is from before it\n"+ + " the whole database: leave out --table, --exclude-table and --schema-only", + option, + ) + } + + if len(values) == 0 { + words = append(words, option) + } + + for _, value := range values { + words = append(words, option+"="+value) + } + + return nil + } + + if p.schemaOnly { + if err := ask("--schema-only"); err != nil { + return "", err + } + } + + if len(p.tables) > 0 { + if err := ask("--table", p.tables...); err != nil { + return "", err + } + } + + if len(p.excluded) > 0 { + if err := ask("--exclude-table", p.excluded...); err != nil { + return "", err + } + } + + return strings.Join(words, " "), nil +} + +// tableList is a flag given more than once, or with commas, or both: +// `--table users --table orders` and `--table users,orders` are one list. +type tableList []string + +func (l *tableList) String() string { + return strings.Join(*l, ",") +} + +func (l *tableList) Set(value string) error { + for _, table := range strings.Split(value, ",") { + if table = strings.TrimSpace(table); table != "" && !slices.Contains(*l, table) { + *l = append(*l, table) + } + } + + return nil +} + +// writeDump runs the dump into a file, which appears only once it is whole. +func writeDump(env *Env, process *exec.Cmd, path string, gzipped bool, named string) error { + if _, err := os.Stat(path); err == nil { + // Asked rather than refused, as Upsun asks: replacing yesterday's + // dump with today's is the ordinary case. With nobody to ask, kept. + if !env.confirm(fmt.Sprintf("%s is already here. Replace it?", path)) { + return fmt.Errorf("%s is already here\n keep both: --timestamp, or name another with --file", path) + } + } + + dir, base := filepath.Split(path) + if dir == "" { + dir = "." + } + + // Beside the final name, so the rename at the end is on one filesystem + // and cannot fail half way. + partial, err := os.CreateTemp(dir, "."+base+".*.partial") + if err != nil { + return fmt.Errorf("cannot write %s: %w", path, err) + } + + finished := false + + defer func() { + if !finished { + partial.Close() + os.Remove(partial.Name()) + } + }() + + var ( + out io.Writer = partial + compressor *gzip.Writer + ) + + if gzipped { + compressor = gzip.NewWriter(partial) + out = compressor + } + + buffered := bufio.NewWriterSize(out, 1<<20) + process.Stdout = buffered + + // To stderr, as every progress line is. + env.Printer.Say("Dumping the database of %s to %s…", named, path) + + if err := runOrPrint(env, process, false); err != nil { + return err + } + + if err := buffered.Flush(); err != nil { + return fmt.Errorf("cannot write %s: %w", path, err) + } + + if compressor != nil { + if err := compressor.Close(); err != nil { + return fmt.Errorf("cannot write %s: %w", path, err) + } + } + + if err := partial.Close(); err != nil { + return fmt.Errorf("cannot write %s: %w", path, err) + } + + if err := os.Rename(partial.Name(), path); err != nil { + return fmt.Errorf("cannot write %s: %w", path, err) + } + + finished = true + + size := "" + if info, err := os.Stat(path); err == nil { + size = " (" + humanBytes(info.Size()) + ")" + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"dump": map[string]any{ + "path": path, + "compressed": gzipped, + }}) + } + + env.Printer.Good("Wrote %s%s", path, size) + + return nil +} + +// streamDump runs the dump onto stdout, compressed where asked. +// +// Refused onto a terminal: a dump is megabytes of SQL that scroll past and +// are gone, which is never what somebody typing it meant. The fix is one of +// two words, and the message says both. +func streamDump(env *Env, process *exec.Cmd, gzipped, dryRun bool, host string) error { + if !dryRun && env.stdoutIsTerminal() { + return Usagef(nil, + "standard output is a terminal, and a dump would be printed into it\n"+ + " redirect it: vallic db export > dump.sql\n"+ + " or let it name a file: vallic db dump") + } + + if dryRun { + if gzipped { + env.Printer.Line("%s | gzip", vssh.Describe(process)) + } else { + env.Printer.Line("%s", vssh.Describe(process)) + } + + return nil + } + + // To stderr, because stdout is the dump. A progress line in the middle + // of a redirect would corrupt the file. + env.Printer.Say("Streaming the database out of %s…", host) + + if !gzipped { + return runOrPrint(env, process, false) + } + + compressor := gzip.NewWriter(env.Out) + process.Stdout = compressor + + if err := runOrPrint(env, process, false); err != nil { + return err + } + + return compressor.Close() +} + +// openDump is what an import reads: a file, or standard input, decompressed +// where it is gzip. +// +// Told apart by the first two bytes rather than by a name, because standard +// input has none and a dump called .sql is sometimes compressed anyway. The +// engine's client would read gzip as a syntax error on line one. +func openDump(env *Env, file string) (reader io.Reader, described string, closer func(), err error) { + source, described, closer := io.Reader(env.In), "standard input", func() {} + + if file != "" { + opened, err := os.Open(file) + if err != nil { + return nil, "", nil, fmt.Errorf("cannot read %s: %w", file, err) + } + + source, described, closer = opened, file, func() { opened.Close() } + } + + buffered := bufio.NewReaderSize(source, 1<<20) + + magic, err := buffered.Peek(2) + if err != nil && !errors.Is(err, io.EOF) { + closer() + + return nil, "", nil, fmt.Errorf("cannot read %s: %w", described, err) + } + + if len(magic) == 2 && magic[0] == 0x1f && magic[1] == 0x8b { + decompressed, err := gzip.NewReader(buffered) + if err != nil { + closer() + + return nil, "", nil, fmt.Errorf("%s looks compressed and is not gzip: %w", described, err) + } + + return decompressed, described + ", decompressed", closer, nil + } + + return buffered, described, closer, nil +} + +// stdoutIsTerminal reports whether what this writes lands on a screen. +func (e *Env) stdoutIsTerminal() bool { + file, ok := e.Out.(*os.File) + + return ok && terminal.IsTerminal(file) +} + +// stdinIsTerminal reports whether stdin is somebody typing, whatever +// --no-input says: the question here is what the bytes are, not whether to +// ask anything. +func (e *Env) stdinIsTerminal() bool { + file, ok := e.In.(*os.File) + + return ok && terminal.IsTerminal(file) +} + +// humanBytes is a size as somebody reads one. +func humanBytes(n int64) string { + const unit = 1024 + + if n < unit { + return fmt.Sprintf("%d B", n) + } + + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + + return fmt.Sprintf("%.1f %cB", float64(n)/float64(div), "KMGTPE"[exp]) +} + +// dbSizeCommand says how much the database holds, and where. +// +// Upsun's db:size, and one step further: the largest tables beside the total, +// because the question after "how big is it" is nearly always "what is it", +// and in a CMS the answer is usually a cache table. +func dbSizeCommand() *Command { + var ( + identity string + dryRun bool + all bool + ) + + return &Command{ + Name: "size", + Summary: "show how much the database holds, and its largest tables", + Usage: "db size [] [--all]", + Long: `The database's size and its ten largest tables, data and indexes +together; --all lists every table. + +On PostgreSQL the total is the database's own size on disk, catalogs and all, +so it is a little more than its tables add up to. MySQL and MariaDB have no +such figure, and there the total is the tables'.`, + Flags: func(fs *flag.FlagSet) { + identityFlag(fs, &identity) + fs.BoolVar(&dryRun, "dry-run", false, "print the ssh command instead of running it") + fs.BoolVar(&all, "all", false, "list every table, not the ten largest") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + target, resolved, err := env.sshTargetFor(ctx, first(args), identity) + if err != nil { + return err + } + + if target.DBCLI == "" { + return fmt.Errorf("this environment runs no database") + } + + if target.DBSize == "" { + return fmt.Errorf("this environment does not report its database's size yet: its machines' agent, or the control plane, is from before db size") + } + + process := target.Verb(target.DBSize, false) + + if dryRun { + env.Printer.Line("%s", vssh.Describe(process)) + + return nil + } + + var answer strings.Builder + process.Stdout = &answer + + if err := runOrPrint(env, process, false); err != nil { + return err + } + + size, err := parseDatabaseSize(answer.String()) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"database": size}) + } + + env.Printer.Line("%s holds %s in %d tables.", resolved.Named(), humanBytes(size.Bytes), len(size.Tables)) + + shown := size.Tables + if !all && len(shown) > 10 { + shown = shown[:10] + } + + if len(shown) == 0 { + return nil + } + + table := output.Table{Columns: []string{"table", "size", "share"}} + + for _, t := range shown { + share := "-" + if size.Bytes > 0 { + share = fmt.Sprintf("%.0f%%", 100*float64(t.Bytes)/float64(size.Bytes)) + } + + table.Rows = append(table.Rows, []string{t.Name, humanBytes(t.Bytes), share}) + } + + env.Printer.Line("") + + if err := env.Printer.Print(table, nil); err != nil { + return err + } + + if rest := len(size.Tables) - len(shown); rest > 0 { + env.Printer.Say(" and %d more: --all lists them", rest) + } + + return nil + }, + } +} + +// DatabaseSize is what db-size answered. +type DatabaseSize struct { + Bytes int64 `json:"bytes"` + Tables []TableSize `json:"tables"` +} + +// TableSize is one table's data and indexes. +type TableSize struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` +} + +// parseDatabaseSize reads db-size's answer: the total alone on the first +// line, then a table and its size on each line after, a tab between, largest +// first. +func parseDatabaseSize(answer string) (*DatabaseSize, error) { + lines := strings.Split(strings.TrimSpace(answer), "\n") + + total, err := strconv.ParseInt(strings.TrimSpace(lines[0]), 10, 64) + if err != nil { + return nil, fmt.Errorf("the database's size came back as %q, which this version cannot read", lines[0]) + } + + size := &DatabaseSize{Bytes: total, Tables: []TableSize{}} + + for _, line := range lines[1:] { + name, bytes, found := strings.Cut(line, "\t") + if !found { + continue + } + + n, err := strconv.ParseInt(strings.TrimSpace(bytes), 10, 64) + if err != nil { + continue + } + + size.Tables = append(size.Tables, TableSize{Name: name, Bytes: n}) + } + + return size, nil +} diff --git a/internal/cli/dump_test.go b/internal/cli/dump_test.go new file mode 100644 index 0000000..434fe6b --- /dev/null +++ b/internal/cli/dump_test.go @@ -0,0 +1,276 @@ +package cli + +import ( + "bytes" + "compress/gzip" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/vallic/vallic-cli/internal/api" + vssh "github.com/vallic/vallic-cli/internal/ssh" +) + +// A dump needs nothing but the environment, so the name has to say whose it +// is: two environments of one project, or two projects' productions, never +// land on the same file. +func TestDumpFilenameNamesTheProjectAndEnvironment(t *testing.T) { + target := &Target{ + Project: &api.Project{MachineName: "acme-site"}, + Environment: &api.Environment{Name: "production", Slug: "acme-site-production"}, + } + at := time.Date(2026, 10, 18, 14, 30, 5, 0, time.FixedZone("CEST", 2*60*60)) + + cases := []struct { + gzipped, stamped bool + want string + }{ + {false, false, "acme-site--production--dump.sql"}, + {true, false, "acme-site--production--dump.sql.gz"}, + // UTC, whatever the machine's zone, so names sort as dumps were taken. + {false, true, "acme-site--production--20261018-123005-UTC--dump.sql"}, + } + + for _, c := range cases { + if got := dumpFilename(target, dumpPart{}, c.gzipped, c.stamped, at); got != c.want { + t.Errorf("dumpFilename(gzip %v, timestamp %v) = %q, want %q", c.gzipped, c.stamped, got, c.want) + } + } + + // An environment named after its branch carries the branch's slash, + // which must not become a directory. + branch := &Target{Project: &api.Project{MachineName: "acme"}, Environment: &api.Environment{Name: "feature/checkout"}} + if got := dumpFilename(branch, dumpPart{}, false, false, at); got != "acme--feature-checkout--dump.sql" { + t.Errorf("dumpFilename() = %q, want no slash in it", got) + } +} + +// The file appears only once the dump is whole. A dump that fails half way +// must leave nothing behind that looks like a good one — not the name, and +// not the partial file beside it. +func TestWriteDumpLeavesNothingWhenTheDumpFails(t *testing.T) { + env, _, _ := followEnv(t) + dir := t.TempDir() + path := filepath.Join(dir, "acme--production--dump.sql") + + failing := exec.Command("sh", "-c", "printf 'CREATE TABLE half'; exit 3") + + err := writeDump(env, failing, path, false, "acme-production") + + var passthrough *PassthroughError + if !errors.As(err, &passthrough) || passthrough.Code != 3 { + t.Fatalf("writeDump() = %v, want the far side's exit status passed through", err) + } + + entries, _ := os.ReadDir(dir) + if len(entries) != 0 { + t.Errorf("left behind %v after a failed dump", names(entries)) + } + + working := exec.Command("sh", "-c", "printf 'CREATE TABLE whole;\\n'") + + if err := writeDump(env, working, path, false, "acme-production"); err != nil { + t.Fatalf("writeDump() = %v", err) + } + + if raw, _ := os.ReadFile(path); string(raw) != "CREATE TABLE whole;\n" { + t.Errorf("the dump is %q", raw) + } + + entries, _ = os.ReadDir(dir) + if len(entries) != 1 { + t.Errorf("the directory holds %v, want the dump alone", names(entries)) + } + + // Nobody to ask, so the dump already there is kept. + again := exec.Command("sh", "-c", "printf 'newer'") + if err := writeDump(env, again, path, false, "acme-production"); err == nil || !strings.Contains(err.Error(), "--timestamp") { + t.Errorf("writeDump() over an existing dump = %v, want a refusal naming --timestamp", err) + } + + if raw, _ := os.ReadFile(path); string(raw) != "CREATE TABLE whole;\n" { + t.Errorf("the refused dump replaced the file: %q", raw) + } +} + +// What db dump --gzip writes, db import reads back, from a file or from +// standard input, without being told it is compressed. +func TestADumpIsReadBackCompressedOrNot(t *testing.T) { + env, _, _ := followEnv(t) + dir := t.TempDir() + path := filepath.Join(dir, "dump.sql.gz") + + if err := writeDump(env, exec.Command("sh", "-c", "printf 'INSERT INTO t VALUES (1);\\n'"), path, true, "acme-production"); err != nil { + t.Fatalf("writeDump() = %v", err) + } + + raw, _ := os.ReadFile(path) + if len(raw) < 2 || raw[0] != 0x1f || raw[1] != 0x8b { + t.Fatalf("the .gz dump is not gzip: %q", raw) + } + + reader, described, closer, err := openDump(env, path) + if err != nil { + t.Fatalf("openDump() = %v", err) + } + defer closer() + + if got, _ := io.ReadAll(reader); string(got) != "INSERT INTO t VALUES (1);\n" { + t.Errorf("read back %q", got) + } + + if !strings.Contains(described, "decompressed") { + t.Errorf("described as %q, want it said that it was decompressed", described) + } + + // The same bytes on standard input. + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + writer.Write([]byte("SELECT 1;\n")) + writer.Close() + + env.In = &compressed + + reader, _, closer, err = openDump(env, "") + if err != nil { + t.Fatalf("openDump(stdin) = %v", err) + } + defer closer() + + if got, _ := io.ReadAll(reader); string(got) != "SELECT 1;\n" { + t.Errorf("read back %q from standard input", got) + } + + // And a plain dump passes through untouched. + env.In = strings.NewReader("SELECT 2;\n") + + reader, described, closer, err = openDump(env, "") + if err != nil { + t.Fatalf("openDump(plain) = %v", err) + } + defer closer() + + if got, _ := io.ReadAll(reader); string(got) != "SELECT 2;\n" || described != "standard input" { + t.Errorf("read back %q as %q", got, described) + } +} + +// --stdout writes no file, so the flags that shape one are refused rather +// than ignored — before anything is asked of the platform. +func TestDumpRefusesFlagsThatContradictEachOther(t *testing.T) { + for _, args := range [][]string{ + {"db", "dump", "--stdout", "--file", "x.sql"}, + {"db", "dump", "-o", "-t"}, + {"db", "dump", "--file", "x.sql", "--directory", "dumps"}, + {"db", "dump", "staging", "production"}, + } { + env, _, _ := followEnv(t) + + err := Execute(t.Context(), Root(), env, args) + + var usage *UsageError + if !errors.As(err, &usage) { + t.Errorf("%q = %v, want a usage error", args, err) + } + } +} + +func names(entries []os.DirEntry) []string { + var out []string + for _, entry := range entries { + out = append(out, entry.Name()) + } + + return out +} + +// A part of the database is asked for only where the environment takes it. +// A machine whose agent reads no options would run `db-export --table=users` +// as a command and answer "not found", so that is refused here, in a sentence. +func TestAPartOfTheDatabaseIsAskedForOnlyWhereItIsOffered(t *testing.T) { + part := dumpPart{tables: tableList{"users", "orders"}, excluded: tableList{"sessions"}, schemaOnly: true} + + old := &vssh.Target{SSHTarget: api.SSHTarget{DBExport: "db-export"}} + if _, err := part.verb(old); err == nil || !strings.Contains(err.Error(), "does not offer --schema-only") { + t.Errorf("verb() on an agent with no options = %v, want a refusal", err) + } + + // The whole database is still there to dump. + if verb, err := (&dumpPart{}).verb(old); err != nil || verb != "db-export" { + t.Errorf("verb() for the whole database = %q, %v", verb, err) + } + + current := &vssh.Target{SSHTarget: api.SSHTarget{ + DBExport: "db-export", + DBExportOptions: []string{"--schema-only", "--table", "--exclude-table"}, + }} + + verb, err := part.verb(current) + if err != nil { + t.Fatalf("verb() = %v", err) + } + + if want := "db-export --schema-only --table=users --table=orders --exclude-table=sessions"; verb != want { + t.Errorf("verb() = %q, want %q", verb, want) + } + + // And the file says which part it holds, as Upsun names one. + target := &Target{Project: &api.Project{MachineName: "acme"}, Environment: &api.Environment{Name: "production"}} + if got := dumpFilename(target, part, true, false, time.Now()); got != "acme--production--users,orders--excl-sessions--schema--dump.sql.gz" { + t.Errorf("dumpFilename() = %q", got) + } +} + +// What the agent would refuse is refused before anything connects. +func TestATableNameIsCheckedBeforeConnecting(t *testing.T) { + for _, part := range []dumpPart{ + {tables: tableList{"cache_*"}}, + {tables: tableList{"-r/tmp/x"}}, + {excluded: tableList{"a;b"}}, + {tables: tableList{"users"}, excluded: tableList{"users"}}, + } { + var usage *UsageError + if err := part.check(); !errors.As(err, &usage) { + t.Errorf("check(%+v) = %v, want a usage error", part, err) + } + } + + var list tableList + list.Set("users, orders") + list.Set("users") + list.Set("public.audit") + + if list.String() != "users,orders,public.audit" { + t.Errorf("tableList = %q, want each once, commas and repeats alike", list.String()) + } + + if err := (&dumpPart{tables: list}).check(); err != nil { + t.Errorf("check() refused %q: %v", list, err) + } +} + +// db-size's answer: the total alone, then a table and its size on each line. +func TestTheDatabaseSizeIsReadAsTheAgentWritesIt(t *testing.T) { + size, err := parseDatabaseSize("104857600\ncache_render\t62914560\nusers\t1048576\n") + if err != nil { + t.Fatal(err) + } + + if size.Bytes != 104857600 || len(size.Tables) != 2 || size.Tables[0].Name != "cache_render" || size.Tables[1].Bytes != 1048576 { + t.Errorf("read as %+v", size) + } + + if _, err := parseDatabaseSize("ERROR 1045 (28000): Access denied\n"); err == nil { + t.Error("an answer that is not a size was read as one") + } + + empty, err := parseDatabaseSize("0\n") + if err != nil || empty.Bytes != 0 || empty.Tables == nil { + t.Errorf("an empty database read as %+v, %v; want zero and an empty list", empty, err) + } +} diff --git a/internal/cli/environment.go b/internal/cli/environment.go index 66abd17..1bc4d81 100644 --- a/internal/cli/environment.go +++ b/internal/cli/environment.go @@ -93,6 +93,7 @@ func envCommand() *Command { envInfoCommand(), envCreateCommand(), envSourceCommand(), + envMaintenanceCommand(), envDeleteCommand(), }, } diff --git a/internal/cli/init.go b/internal/cli/init.go index 28567f4..14ff480 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" ) @@ -28,6 +29,8 @@ const ( typeGo = "go" typePython = "python" typeDjango = "django" + typeRuby = "ruby" + typeRust = "rust" ) // initCommand writes a starting vallic.yaml. @@ -52,7 +55,7 @@ func initCommand() *Command { return &Command{ Name: "init", Summary: "write a starting vallic.yaml for this checkout", - Usage: "init [--type drupal|laravel|wordpress|php|nodejs|go|python|django] [--force]", + Usage: "init [--type drupal|laravel|wordpress|php|nodejs|go|rust|python|django|ruby] [--force]", Long: `Writes vallic.yaml in the working directory, and nothing else. It needs no credential and reaches nothing, so it works in a checkout of a project that does not exist on the platform yet. @@ -60,9 +63,10 @@ project that does not exist on the platform yet. What kind of project this is comes from the files here: a composer.json requiring drupal/core is Drupal, laravel/framework is Laravel, a WordPress core package or a wp-config.php is WordPress, a composer.json on its own is -PHP, go.mod is Go, manage.py or a dependency on Django is Django, a uv.lock, -pyproject.toml or requirements.txt is otherwise Python, and package.json is -Node. The command says which file decided it, and --type overrides the +PHP, go.mod is Go, Cargo.toml is Rust, manage.py or a dependency on Django is +Django, a uv.lock, pyproject.toml or requirements.txt is otherwise Python, a +Gemfile is Ruby, and package.json is Node — run on Bun where a bun.lock sits +beside it. The command says which file decided it, and --type overrides the answer. The kind is worth getting right rather than settling for the generic one: it @@ -182,6 +186,11 @@ type starter struct { Build []string Deploy []string + // Cache is what the build keeps between builds, as the control plane's + // ManifestSuggestion::cachePaths() names it for the same kind. Written + // only beside a build, since it is a key of the build. + Cache []string + // Start is what serves the application, for a runtime that is its own // server. Start string @@ -240,24 +249,77 @@ func detectProject(dir string) (kind, because string, err error) { return typeGo, "go.mod", nil } + if _, readErr := os.Stat(filepath.Join(dir, "Cargo.toml")); readErr == nil { + return typeRust, "Cargo.toml", nil + } + // Before package.json, for the reason composer.json is first: a Django // site builds its front end with npm and is not a Node project. if kind, because := detectPython(dir); kind != "" { return kind, because, nil } + // Before package.json too: a Rails application bundling its JavaScript + // with jsbundling-rails carries one. + if because := detectRuby(dir); because != "" { + return typeRuby, because, nil + } + if _, readErr := os.Stat(filepath.Join(dir, "package.json")); readErr == nil { + // Still Node: Bun is a runtime of the nodejs type, chosen under + // `runtime` and not by the type. Named, because it changes every + // step the file writes. + if lock := bunLockfile(dir); lock != "" { + return typeNode, "package.json, with " + lock + " beside it, so on Bun", nil + } + return typeNode, "package.json", nil } // A usage error rather than a failure: the fix is an argument, and the // help printed with it lists the kinds that work. return "", "", Usagef(nil, - "nothing here says what this project is: no composer.json, wp-config.php, go.mod, pyproject.toml, requirements.txt or package.json\n"+ - " name it: vallic init --type drupal|laravel|wordpress|php|nodejs|go|python|django", + "nothing here says what this project is: no composer.json, wp-config.php, go.mod, Cargo.toml, pyproject.toml, requirements.txt, Gemfile or package.json\n"+ + " name it: vallic init --type drupal|laravel|wordpress|php|nodejs|go|rust|python|django|ruby", ) } +// detectRuby recognises a Ruby project, and says whether it is Rails. +// +// One type for both, as on the control plane, where Ruby is Rails's type and +// any other Rack application runs on it unchanged. Rails is still worth +// naming in the reason: it is what decides the asset and migration steps. +func detectRuby(dir string) (because string) { + raw, err := os.ReadFile(filepath.Join(dir, "Gemfile")) + if err != nil { + return "" + } + + if lock, err := os.ReadFile(filepath.Join(dir, "Gemfile.lock")); err == nil && rubyRequires(lock, "rails") { + return "Gemfile.lock, which requires Rails" + } + + if rubyRequires(raw, "rails") { + return "Gemfile, which requires Rails" + } + + return "Gemfile" +} + +// bunLockfile is the Bun lockfile in dir, or empty where there is none. +// +// Text since Bun 1.2 and binary before it; the control plane looks for both +// when it asks whether a Bun build was declared. +func bunLockfile(dir string) string { + for _, name := range []string{"bun.lock", "bun.lockb"} { + if _, err := os.Stat(filepath.Join(dir, name)); err == nil { + return name + } + } + + return "" +} + // The files a Python project declares its dependencies in, the most exact // first: a lock names every package, a requirements file may only include // another. @@ -293,11 +355,11 @@ func detectPython(dir string) (kind, because string) { // knownProjectType refuses a --type this writes no file for. func knownProjectType(kind string) error { switch kind { - case typeDrupal, typeLaravel, typeWordPress, typePHP, typeNode, typeGo, typePython, typeDjango: + case typeDrupal, typeLaravel, typeWordPress, typePHP, typeNode, typeGo, typeRust, typePython, typeDjango, typeRuby: return nil } - return Usagef(nil, "--type takes drupal, laravel, wordpress, php, nodejs, go, python or django, not %q", kind) + return Usagef(nil, "--type takes drupal, laravel, wordpress, php, nodejs, go, rust, python, django or ruby, not %q", kind) } // starterFor builds the starting file for one kind of project. @@ -307,8 +369,12 @@ func starterFor(kind, dir string) *starter { return nodeStarter(dir) case typeGo: return goStarter(dir) + case typeRust: + return rustStarter(dir) case typePython, typeDjango: return pythonStarter(kind, dir) + case typeRuby: + return rubyStarter(dir) default: return phpStarter(kind, dir) } @@ -335,9 +401,24 @@ func phpStarter(kind, dir string) *starter { // declared is a deploy the control plane stops, naming the command it // expected to find here. Build: []string{"composer install --no-dev --optimize-autoloader"}, + Cache: []string{"vendor"}, Services: []string{"mariadb: '11.8'"}, } + // Everything the install writes, not just vendor/: composer puts Drupal + // core and contrib under web/, and a build that cached vendor alone still + // extracted all of core into an empty web/core on every push. + if kind == typeDrupal { + manifest.Cache = []string{ + "vendor", + "web/core", + "web/modules/contrib", + "web/themes/contrib", + "web/profiles/contrib", + "web/libraries", + } + } + // What brings a release up to date with the code in it. Each is the // framework's own command, run by the platform inside the application // container, and each is idempotent: a deploy that runs twice is a deploy @@ -367,6 +448,7 @@ func phpStarter(kind, dir string) *starter { // wrong. `composer install` with no composer.json fails the build, and // it fails it at the point where there is nothing to roll back to. manifest.Build = nil + manifest.Cache = nil manifest.Notes = append(manifest.Notes, "no composer.json here, so no build step is written and the repository is deployed as it is") } @@ -380,6 +462,10 @@ func phpStarter(kind, dir string) *starter { func nodeStarter(dir string) *starter { raw, readErr := os.ReadFile(filepath.Join(dir, "package.json")) + if bunLockfile(dir) != "" { + return bunStarter(raw, readErr) + } + manifest := &starter{ Type: typeNode, VersionKey: "node", @@ -389,6 +475,7 @@ func nodeStarter(dir string) *starter { // with nothing in the file to say that it did. Version: firstVersion(packageEngine(raw, "node"), 1), Build: []string{"npm ci"}, + Cache: []string{"node_modules"}, Services: []string{"postgres: '18'"}, } @@ -396,6 +483,10 @@ func nodeStarter(dir string) *starter { manifest.Build = append(manifest.Build, "npm run build") } + // node_modules is the runtime, not a build input, and travels in the + // release, so the last step leaves only what production needs. + manifest.Build = append(manifest.Build, "npm prune --omit=dev") + // Left out of the file rather than guessed at. The image runs `npm start` // where nothing says otherwise, and inventing an entry point here would // write a `start:` line naming a file that may not exist. Only where the @@ -411,6 +502,74 @@ func nodeStarter(dir string) *starter { return manifest } +// bunStarter is a Node application run by Bun, in Node's place. +// +// The same type, port and node_modules as Node, with Bun as the runtime and +// the package manager. What says so is `bun:` under runtime rather than +// `node:`, so it is written whether or not a version is known: without the +// key the project would build and run on Node. +func bunStarter(raw []byte, readErr error) *starter { + manifest := &starter{ + Type: typeNode, + VersionKey: "bun", + // Two components. The Bun images are tagged `1.4.3-slim-1`, and the + // catalogue matches a manifest's `1.4` on that line. + Version: firstVersion(firstNonEmpty(packageEngine(raw, "bun"), packageManagerVersion(raw, "bun")), 2), + Build: []string{"bun install --frozen-lockfile"}, + Cache: []string{"node_modules"}, + // postgres because Bun.sql reads the DATABASE_URL the platform sets + // for it, with nothing to configure. + Services: []string{"postgres: '18'"}, + } + + if hasScript(raw, "build") { + manifest.Build = append(manifest.Build, "bun run build") + } + + // Bun has no prune. The production install is made again, from Bun's own + // download cache, into a node_modules with nothing else in it. + manifest.Build = append(manifest.Build, "rm -rf node_modules && bun install --frozen-lockfile --production") + + if readErr == nil && !hasScript(raw, "start") { + manifest.Notes = append(manifest.Notes, + "package.json has no start script, so add a start: line naming what serves the application, such as bun index.ts") + } + + // The one runtime written without the repository stating it, because a + // key with no version is not asking for Bun at all: the control plane + // would build on Node. The major is the one every Bun lockfile is + // written by, and it matches whichever 1.x the platform offers. + if manifest.Version == "" { + manifest.Version = "1" + manifest.Notes = append(manifest.Notes, + "nothing here states a Bun version — engines.bun or packageManager in package.json — so runtime names Bun 1, which is what chooses Bun over Node; pin the minor you test with") + } + + return manifest +} + +// packageManagerVersion reads the version a package.json's packageManager +// field pins for one tool: `bun@1.4.3` is 1.4.3 for bun. +func packageManagerVersion(raw []byte, tool string) string { + var file struct { + PackageManager string `json:"packageManager"` + } + + if err := json.Unmarshal(raw, &file); err != nil { + return "" + } + + name, version, found := strings.Cut(file.PackageManager, "@") + if !found || name != tool { + return "" + } + + // Corepack appends a hash, `bun@1.4.3+sha512.…`. + version, _, _ = strings.Cut(version, "+") + + return version +} + // goStarter is a compiled Go binary, which serves its own HTTP too. func goStarter(dir string) *starter { raw, readErr := os.ReadFile(filepath.Join(dir, "go.mod")) @@ -441,6 +600,174 @@ func goStarter(dir string) *starter { return manifest } +// rustStarter is a compiled Rust binary, which serves its own HTTP like Go's. +// +// cargo installs it into the release's bin/, which is on PATH, and the image +// starts bin/server. Nothing is cached: cargo's registry and build directory +// are kept by the platform outside the checkout, and target/ declared as a +// cache would be copied into every release. +func rustStarter(dir string) *starter { + raw, readErr := os.ReadFile(filepath.Join(dir, "Cargo.toml")) + + manifest := &starter{ + Type: typeRust, + VersionKey: "rust", + Version: rustVersion(dir), + Build: []string{"cargo install --locked --path . --root ."}, + Services: []string{"postgres: '18'"}, + } + + // --locked builds what Cargo.lock says, and with no Cargo.lock there is + // nothing to hold it to. The same bargain as uv without its lock. + if _, err := os.Stat(filepath.Join(dir, "Cargo.lock")); err != nil { + manifest.Build = []string{"cargo install --path . --root ."} + manifest.Notes = append(manifest.Notes, + "no Cargo.lock here, so every build resolves the dependencies again: run cargo generate-lockfile, commit it, and add --locked") + } + + // The binary is named after the package, or its [[bin]]: written out + // rather than left to the image's bin/server, because a package called + // anything else serves nothing. + switch binaries := rustBinaries(dir, raw); len(binaries) { + case 0: + manifest.Notes = append(manifest.Notes, + "Cargo.toml here builds no binary — a workspace, or a library — so check which package the build step names and add a start: line") + case 1: + manifest.Start = "bin/" + binaries[0] + default: + manifest.Notes = append(manifest.Notes, fmt.Sprintf( + "Cargo.toml builds %s, so add a start: line naming the one that serves, as bin/", + strings.Join(binaries, ", "), + )) + } + + if manifest.Version == "" && readErr == nil { + manifest.Notes = append(manifest.Notes, + "nothing here states a Rust version — channel in rust-toolchain.toml — so none is pinned and the platform's default runs") + } + + return manifest +} + +// rustVersion is the toolchain the repository builds with. +// +// rust-toolchain.toml's channel, or the older rust-toolchain file, which the +// platform's build does not obey: the version it builds with is the one under +// runtime, so this is where that file's answer has to go. A channel such as +// stable or nightly names no version and pins nothing. Cargo.toml's +// rust-version is not read: it is the oldest the crate supports, and the +// platform offers the newest two. +func rustVersion(dir string) string { + if raw, err := os.ReadFile(filepath.Join(dir, "rust-toolchain.toml")); err == nil { + if channel := tomlValue(raw, "toolchain", "channel"); startsWithDigit(channel) { + return firstVersion(channel, 2) + } + + return "" + } + + if raw, err := os.ReadFile(filepath.Join(dir, "rust-toolchain")); err == nil { + line, _, _ := strings.Cut(strings.TrimSpace(string(raw)), "\n") + + if channel := strings.TrimSpace(line); startsWithDigit(channel) { + return firstVersion(channel, 2) + } + } + + return "" +} + +// rustBinaries names what `cargo install` puts in bin/. +// +// Each [[bin]], the package itself where src/main.rs exists, and each file +// under src/bin/, which cargo builds without being told. +func rustBinaries(dir string, raw []byte) []string { + var names []string + + add := func(name string) { + if name != "" && !slices.Contains(names, name) { + names = append(names, name) + } + } + + for _, name := range tomlTables(raw, "[bin]", "name") { + add(name) + } + + if _, err := os.Stat(filepath.Join(dir, "src", "main.rs")); err == nil { + add(tomlValue(raw, "package", "name")) + } + + if entries, err := os.ReadDir(filepath.Join(dir, "src", "bin")); err == nil { + for _, entry := range entries { + switch { + case !entry.IsDir() && strings.HasSuffix(entry.Name(), ".rs"): + add(strings.TrimSuffix(entry.Name(), ".rs")) + case entry.IsDir(): + if _, err := os.Stat(filepath.Join(dir, "src", "bin", entry.Name(), "main.rs")); err == nil { + add(entry.Name()) + } + } + } + } + + return names +} + +// tomlValue reads one string out of one table of a TOML file. +// +// Enough of TOML for the two files read here, which keep the values asked +// about as `key = "value"` under a plain [table] header, and no more: this is +// a starting file, and a value this cannot find is said rather than guessed. +func tomlValue(raw []byte, table, key string) string { + values := tomlTables(raw, table, key) + if len(values) == 0 { + return "" + } + + return values[0] +} + +// tomlTables reads a key out of every table with a header, in order: once +// for [package], once per entry of an array of tables such as [[bin]]. +// +// The table is named as its header is written, without the outer brackets: +// `package` for [package], `[bin]` for [[bin]]. +func tomlTables(raw []byte, table, key string) []string { + var ( + values []string + current string + ) + + for _, line := range strings.Split(string(raw), "\n") { + line = strings.TrimSpace(line) + + if strings.HasPrefix(line, "[") { + header, _, _ := strings.Cut(line, "#") + current = strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(header), "["), "]") + continue + } + + if current != table { + continue + } + + name, value, found := strings.Cut(line, "=") + if !found || strings.TrimSpace(name) != key { + continue + } + + value, _, _ = strings.Cut(strings.TrimSpace(value), "#") + values = append(values, strings.Trim(strings.TrimSpace(value), `"'`)) + } + + return values +} + +func startsWithDigit(s string) bool { + return s != "" && s[0] >= '0' && s[0] <= '9' +} + // pythonStarter is a Python application, Django or not, which serves its own // HTTP through Gunicorn. // @@ -465,14 +792,18 @@ func pythonStarter(kind, dir string) *starter { // Whatever the repository locks with. uv with its lock, so a build // installs exactly what was tested; pip where requirements.txt is what // the project keeps; uv without a lock, resolving afresh, for a - // pyproject.toml alone. + // pyproject.toml alone. Each keeps .venv between builds, so the next one + // only brings it up to date. switch { case exists("uv.lock"): manifest.Build = []string{"uv sync --locked --no-dev --no-editable"} + manifest.Cache = []string{".venv"} case exists("requirements.txt"): manifest.Build = []string{"python -m venv .venv", ".venv/bin/pip install -r requirements.txt"} + manifest.Cache = []string{".venv"} case exists("pyproject.toml"): manifest.Build = []string{"uv sync --no-dev --no-editable"} + manifest.Cache = []string{".venv"} manifest.Notes = append(manifest.Notes, "no uv.lock here, so every build resolves the dependencies again: run uv lock, commit it, and add --locked") default: @@ -521,6 +852,202 @@ func pythonStarter(kind, dir string) *starter { return manifest } +// rubyStarter is a Ruby application served by Puma from its release: Rails, +// or Sinatra, Hanami or any other Rack application. +// +// The bundle is installed into the checkout's vendor/bundle, held to +// Gemfile.lock and without the development and test groups — the image's own +// Bundler settings, so `bundle install` is the whole of the step — and travels +// in the release as node_modules does. With no start: line the image runs +// `bundle exec puma -p $PORT`, which reads config/puma.rb. +func rubyStarter(dir string) *starter { + gemfile, gemfileErr := os.ReadFile(filepath.Join(dir, "Gemfile")) + lock, lockErr := os.ReadFile(filepath.Join(dir, "Gemfile.lock")) + + manifest := &starter{ + Type: typeRuby, + VersionKey: "ruby", + Version: rubyVersion(dir, gemfile, lock), + Build: []string{"bundle install"}, + Cache: []string{"vendor/bundle"}, + Services: []string{"postgres: '18'"}, + } + + requires := func(name string) bool { + return rubyRequires(lock, name) || rubyRequires(gemfile, name) + } + + exists := func(name string) bool { + _, err := os.Stat(filepath.Join(dir, name)) + + return err == nil + } + + switch { + case gemfileErr != nil: + // Only through --type: a checkout with no Gemfile is not detected + // as Ruby. `bundle install` with nothing to install fails the build. + manifest.Build, manifest.Cache = nil, nil + manifest.Notes = append(manifest.Notes, + "no Gemfile here, so no build step is written and the repository is deployed as it is") + case lockErr != nil: + // The image installs held to the lock, so a build with none fails + // rather than resolving afresh. + manifest.Notes = append(manifest.Notes, + "no Gemfile.lock here, and the build installs only what it locks: run bundle install and commit Gemfile.lock") + default: + if platforms := gemPlatforms(lock); len(platforms) > 0 && !slices.ContainsFunc(platforms, func(p string) bool { + return p == "ruby" || strings.Contains(p, "linux") + }) { + // A lock made on a Mac names the Mac alone, and an install held + // to it refuses to add the platform it is building on. + manifest.Notes = append(manifest.Notes, fmt.Sprintf( + "Gemfile.lock is locked for %s only, and the build runs on Linux: bundle lock --add-platform x86_64-linux", + strings.Join(platforms, ", "), + )) + } + } + + if manifest.Version == "" { + manifest.Notes = append(manifest.Notes, + "nothing here states a Ruby version — .ruby-version, or ruby in the Gemfile — so none is pinned and the platform's default runs") + } + + if !requires("puma") { + manifest.Notes = append(manifest.Notes, + "the image serves with Puma, which the Gemfile does not name: add gem \"puma\", or a start: line naming what serves the application") + } + + if !requires("rails") { + if !exists("config.ru") { + manifest.Notes = append(manifest.Notes, + "no config.ru here, which is what Puma serves: add one, or a start: line naming what serves the application") + } + + return manifest + } + + // Compiled in the build, which can write, and served by Rails from the + // release. A build is given no environment, and SECRET_KEY_BASE_DUMMY is + // what lets Rails 7.1 and later boot for it without the key. Only with + // an asset pipeline in the bundle: an API application has no + // assets:precompile, and the step would fail the build. + if requires("propshaft") || requires("sprockets-rails") || requires("sprockets") { + manifest.Build = append(manifest.Build, "SECRET_KEY_BASE_DUMMY=1 bin/rails assets:precompile") + + for _, bundler := range []string{"jsbundling-rails", "cssbundling-rails"} { + if requires(bundler) { + manifest.Notes = append(manifest.Notes, + bundler+" builds with Node, which the Ruby image does not have: name node under services and build in its image before assets:precompile, as the Rails guide shows") + + break + } + } + } else { + manifest.Notes = append(manifest.Notes, + "no asset pipeline in the Gemfile — Propshaft or Sprockets — so no assets:precompile step is written") + } + + // db:prepare, not db:migrate: it creates the schema on the empty database + // of a first release, which is when the control plane runs deploy steps + // for Ruby, and migrates every release after. + manifest.Deploy = []string{"'bin/rails db:prepare'"} + + return manifest +} + +// rubyVersion is the Ruby the repository says it runs on. +// +// .ruby-version first, which rbenv, chruby and Rails's own generator write; +// then the Gemfile's `ruby` line; then the version Gemfile.lock was resolved +// with. The image's Ruby is the one that runs whatever these say, so a version +// one of them names is a version runtime has to be told. Two components: +// the images are tagged `4.0.7-slim-trixie-1`, and the catalogue matches +// `4.0` on that line. +func rubyVersion(dir string, gemfile, lock []byte) string { + if raw, err := os.ReadFile(filepath.Join(dir, ".ruby-version")); err == nil { + line, _, _ := strings.Cut(strings.TrimSpace(string(raw)), "\n") + + if version := firstVersion(line, 2); strings.Contains(version, ".") { + return version + } + } + + for _, line := range strings.Split(string(gemfile), "\n") { + line, _, _ = strings.Cut(strings.TrimSpace(line), "#") + + // `ruby "3.4.1"` and `ruby '~> 3.4'`, but not `ruby file: + // ".ruby-version"`, which names a file read above. + if rest, found := strings.CutPrefix(line, "ruby "); found && !strings.Contains(rest, "file:") { + if version := firstVersion(rest, 2); strings.Contains(version, ".") { + return version + } + } + } + + // RUBY VERSION, then ` ruby 3.4.1p194` on the line after it. + if _, after, found := strings.Cut(string(lock), "RUBY VERSION\n"); found { + line, _, _ := strings.Cut(after, "\n") + + if version := firstVersion(line, 2); strings.Contains(version, ".") { + return version + } + } + + return "" +} + +// rubyRequires reports whether a Gemfile or Gemfile.lock names a gem. +// +// A Gemfile names one as `gem "rails", "~> 8.1"`, a lock as ` rails +// (8.1.0)` among its specs and ` rails (~> 8.1)` among its dependencies: +// in both, the name is the first word of the line, quoted in a Gemfile. +func rubyRequires(raw []byte, name string) bool { + for _, line := range strings.Split(string(raw), "\n") { + line, _, _ = strings.Cut(strings.TrimSpace(line), "#") + + if rest, found := strings.CutPrefix(line, "gem "); found { + rest = strings.TrimSpace(rest) + + if len(rest) > 1 && (rest[0] == '"' || rest[0] == '\'') { + if gem, _, closed := strings.Cut(rest[1:], rest[:1]); closed && gem == name { + return true + } + } + + continue + } + + word := strings.FieldsFunc(line, func(r rune) bool { return r == ' ' || r == '(' || r == '!' }) + if len(word) > 0 && word[0] == name { + return true + } + } + + return false +} + +// gemPlatforms are the platforms a Gemfile.lock was resolved for. +func gemPlatforms(lock []byte) []string { + _, after, found := strings.Cut(string(lock), "PLATFORMS\n") + if !found { + return nil + } + + var platforms []string + + for _, line := range strings.Split(after, "\n") { + // The section ends at the first line that is not indented under it. + if !strings.HasPrefix(line, " ") { + break + } + + platforms = append(platforms, strings.TrimSpace(line)) + } + + return platforms +} + // The settings module manage.py names, as startproject writes it. var djangoSettingsModule = regexp.MustCompile(`DJANGO_SETTINGS_MODULE['"]\s*,\s*['"]([A-Za-z_][\w.]*)['"]`) @@ -697,6 +1224,16 @@ func renderStarter(s *starter) string { for _, step := range s.Build { fmt.Fprintf(&b, " - %s\n", step) } + + // Declared whole from the first build: the paths describe one + // install between them, and adding one later starts the set again. + if len(s.Cache) > 0 { + b.WriteString(" cache:\n") + + for _, path := range s.Cache { + fmt.Fprintf(&b, " - %s\n", path) + } + } } if len(s.Deploy) > 0 { diff --git a/internal/cli/init_test.go b/internal/cli/init_test.go index 2c41197..cc791f4 100644 --- a/internal/cli/init_test.go +++ b/internal/cli/init_test.go @@ -112,6 +112,34 @@ func TestDetectProjectReadsTheFilesInTheCheckout(t *testing.T) { }, want: typeGo, }, + { + name: "a rust service with a front end beside it", + files: map[string]string{ + "Cargo.toml": "[package]\nname = \"server\"\n", + "package.json": `{"scripts":{"build":"vite build"}}`, + }, + want: typeRust, + }, + { + name: "a rails application bundling its javascript", + files: map[string]string{ + "Gemfile": "gem \"rails\", \"~> 8.1\"\n", + "Gemfile.lock": "GEM\n specs:\n rails (8.1.0)\n", + "package.json": `{"scripts":{"build":"esbuild app/javascript/*.*"}}`, + }, + want: typeRuby, + }, + { + name: "a rack application, which is ruby too", + files: map[string]string{"Gemfile": "source \"https://rubygems.org\"\ngem \"sinatra\"\n"}, + want: typeRuby, + }, + { + // Bun is a runtime of the nodejs type, not a type of its own. + name: "node on bun", + files: map[string]string{"package.json": `{}`, "bun.lock": "{}"}, + want: typeNode, + }, } for _, c := range cases { @@ -213,6 +241,55 @@ func TestStarterPinsOnlyWhatTheRepositoryStates(t *testing.T) { want: "", wantsNote: true, }, + { + name: "rust, from the toolchain the repository builds with", + kind: typeRust, + files: map[string]string{ + "Cargo.toml": "[package]\nname = \"server\"\nrust-version = \"1.80\"\n", + "rust-toolchain.toml": "[toolchain]\nchannel = \"1.98.1\"\ncomponents = [\"clippy\"]\n", + }, + key: "rust", + want: "1.98", + }, + { + // rust-version is the oldest the crate supports, which the + // platform does not offer: pinning it would run the default + // while the file said otherwise. + name: "rust on a channel, which names no version", + kind: typeRust, + files: map[string]string{"Cargo.toml": "[package]\nname = \"server\"\nrust-version = \"1.80\"\n", "rust-toolchain.toml": "[toolchain]\nchannel = \"stable\"\n"}, + key: "rust", + want: "", + wantsNote: true, + }, + { + name: "ruby, from .ruby-version as rbenv writes it", + kind: typeRuby, + files: map[string]string{".ruby-version": "ruby-3.4.2\n", "Gemfile": "ruby \"3.3.0\"\n"}, + key: "ruby", + want: "3.4", + }, + { + name: "ruby, from the version the lock was resolved with", + kind: typeRuby, + files: map[string]string{"Gemfile": "gem \"puma\"\n", "Gemfile.lock": "GEM\n specs:\n\nRUBY VERSION\n ruby 4.0.7p0\n"}, + key: "ruby", + want: "4.0", + }, + { + name: "bun, from engines, under its own key", + kind: typeNode, + files: map[string]string{"package.json": `{"engines":{"bun":">=1.4.3"}}`, "bun.lock": "{}"}, + key: "bun", + want: "1.4", + }, + { + name: "bun, from the version corepack pins", + kind: typeNode, + files: map[string]string{"package.json": `{"packageManager":"bun@1.3.14+sha512.0123"}`, "bun.lockb": ""}, + key: "bun", + want: "1.3", + }, { name: "nothing at all, which is said out loud", kind: typePHP, @@ -278,20 +355,26 @@ var manifestKeys = map[string]bool{ "version": true, "type": true, "runtime": true, "services": true, "cron": true, "workers": true, "build": true, "deploy": true, "health": true, "env": true, "mounts": true, "start": true, "port": true, + "extra": true, "backup": true, } func TestStarterUsesOnlyKeysTheControlPlaneUnderstands(t *testing.T) { - for _, kind := range []string{typeDrupal, typePHP, typeNode, typeGo, typePython, typeDjango} { + for _, kind := range []string{typeDrupal, typePHP, typeNode, typeGo, typeRust, typePython, typeDjango, typeRuby} { t.Run(kind, func(t *testing.T) { rendered := renderStarter(starterFor(kind, checkoutWith(t, map[string]string{ "composer.json": `{"require":{"php":"^8.4","drupal/core":"^11"}}`, "package.json": `{"engines":{"node":"24"},"scripts":{"build":"next build","start":"next start"}}`, "go.mod": "module example.com/api\n\ngo 1.27\n", + "Cargo.toml": "[package]\nname = \"server\"\n", + "Cargo.lock": "version = 4\n", + "src/main.rs": "fn main() {}\n", "pyproject.toml": "[project]\nrequires-python = \">=3.14\"\n", "uv.lock": "version = 1\n", "manage.py": `os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")`, "mysite/wsgi.py": "", "mysite/settings.py": "STATIC_ROOT = BASE_DIR / 'staticfiles'\n", + "Gemfile": "gem \"rails\"\ngem \"propshaft\"\ngem \"puma\"\n", + "Gemfile.lock": "PLATFORMS\n x86_64-linux\n\nRUBY VERSION\n ruby 4.0.7p0\n", }))) for _, line := range strings.Split(rendered, "\n") { @@ -625,3 +708,227 @@ func TestDjangoStarterServesTheProjectManagePyNames(t *testing.T) { t.Errorf("djangoProject() = %q from two candidates, want none picked", ambiguous) } } + +// The image starts bin/server and cargo names the binary after its package, +// so a package called anything else serves nothing unless the file says what +// to start. Several binaries, or none at the root, are the repository's to +// choose between. +func TestRustStarterStartsTheBinaryCargoInstalls(t *testing.T) { + cases := []struct { + name string + files map[string]string + start string + wantsNote string + }{ + { + name: "the package's own binary", + files: map[string]string{"Cargo.toml": "[package]\nname = \"shop-api\"\n", "src/main.rs": "", "Cargo.lock": ""}, + start: "bin/shop-api", + }, + { + name: "a [[bin]] named for itself", + files: map[string]string{"Cargo.toml": "[package]\nname = \"shop\"\n\n[[bin]] # the only one\nname = \"server\"\npath = \"src/server.rs\"\n", "Cargo.lock": ""}, + start: "bin/server", + }, + { + name: "several, which are the repository's to choose between", + files: map[string]string{ + "Cargo.toml": "[package]\nname = \"shop\"\n", + "src/main.rs": "", + "src/bin/migrate.rs": "", + "src/bin/worker/main.rs": "", + "Cargo.lock": "", + }, + wantsNote: "shop, migrate, worker", + }, + { + name: "a workspace, with nothing to build at the root", + files: map[string]string{"Cargo.toml": "[workspace]\nmembers = [\"api\", \"core\"]\n", "Cargo.lock": ""}, + wantsNote: "builds no binary", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := starterFor(typeRust, checkoutWith(t, c.files)) + + if got.Start != c.start { + t.Errorf("Start = %q, want %q", got.Start, c.start) + } + + if c.wantsNote != "" && !strings.Contains(strings.Join(got.Notes, "\n"), c.wantsNote) { + t.Errorf("Notes = %q, want one mentioning %q", got.Notes, c.wantsNote) + } + + // target/ under the cache would be copied into every release, + // and cargo's own caches are kept by the platform. + if len(got.Cache) != 0 { + t.Errorf("Cache = %v, want nothing for Rust", got.Cache) + } + }) + } + + // --locked with no Cargo.lock has nothing to hold the build to. + unlocked := starterFor(typeRust, checkoutWith(t, map[string]string{"Cargo.toml": "[package]\nname = \"server\"\n", "src/main.rs": ""})) + + if strings.Contains(strings.Join(unlocked.Build, " "), "--locked") { + t.Errorf("Build = %v with no Cargo.lock, want no --locked", unlocked.Build) + } + + if !strings.Contains(strings.Join(unlocked.Notes, "\n"), "Cargo.lock") { + t.Errorf("Notes = %q, want one asking for Cargo.lock to be committed", unlocked.Notes) + } +} + +// Rails's build and deploy, and only what the bundle supports: an API +// application has no assets:precompile to run, and db:prepare is what makes +// the schema on the empty database of a first release. +func TestRailsStarterCompilesAssetsAndPreparesTheDatabase(t *testing.T) { + lock := "GEM\n remote: https://rubygems.org/\n specs:\n propshaft (1.2.0)\n puma (6.6.0)\n rails (8.1.0)\n railties (= 8.1.0)\n\nPLATFORMS\n arm64-darwin-24\n x86_64-linux\n\nDEPENDENCIES\n propshaft\n puma\n rails (~> 8.1)\n" + + got := starterFor(typeRuby, checkoutWith(t, map[string]string{ + "Gemfile": "gem \"rails\", \"~> 8.1\"\ngem \"propshaft\"\ngem \"puma\"\n", + "Gemfile.lock": lock, + "config.ru": "", + ".ruby-version": "4.0.7\n", + })) + + if want := []string{"bundle install", "SECRET_KEY_BASE_DUMMY=1 bin/rails assets:precompile"}; strings.Join(got.Build, " | ") != strings.Join(want, " | ") { + t.Errorf("Build = %q, want %q", got.Build, want) + } + + if len(got.Deploy) != 1 || !strings.Contains(got.Deploy[0], "bin/rails db:prepare") { + t.Errorf("Deploy = %v, want db:prepare", got.Deploy) + } + + if strings.Join(got.Cache, " ") != "vendor/bundle" { + t.Errorf("Cache = %v, want the bundle the release carries", got.Cache) + } + + // Puma serves from the release with no start: line written. + if got.Start != "" { + t.Errorf("Start = %q, want none: the image runs Puma", got.Start) + } + + // Locked for Linux as well as the Mac, so nothing to say about it. + if len(got.Notes) != 0 { + t.Errorf("Notes = %q, want none for a Rails application that has everything", got.Notes) + } + + api := starterFor(typeRuby, checkoutWith(t, map[string]string{ + "Gemfile": "gem \"rails\"\ngem \"puma\"\n", + "Gemfile.lock": "GEM\n specs:\n puma (6.6.0)\n rails (8.1.0)\n\nPLATFORMS\n arm64-darwin-24\n", + })) + + if strings.Contains(strings.Join(api.Build, " "), "assets:precompile") { + t.Errorf("Build = %v, want no assets step without Propshaft or Sprockets", api.Build) + } + + notes := strings.Join(api.Notes, "\n") + + for _, want := range []string{"no asset pipeline", "bundle lock --add-platform x86_64-linux", "arm64-darwin-24"} { + if !strings.Contains(notes, want) { + t.Errorf("Notes = %q, want one mentioning %q", api.Notes, want) + } + } +} + +// A Rack application that is not Rails gets the bundle and nothing else, and +// is told what Puma needs to serve it. +func TestRackStarterSaysWhatPumaNeeds(t *testing.T) { + got := starterFor(typeRuby, checkoutWith(t, map[string]string{"Gemfile": "gem 'sinatra'\n"})) + + if strings.Join(got.Build, " | ") != "bundle install" || len(got.Deploy) != 0 { + t.Errorf("Build = %q, Deploy = %q, want the bundle and no deploy step", got.Build, got.Deploy) + } + + notes := strings.Join(got.Notes, "\n") + + for _, want := range []string{"Gemfile.lock", "Puma", "config.ru"} { + if !strings.Contains(notes, want) { + t.Errorf("Notes = %q, want one mentioning %q", got.Notes, want) + } + } +} + +// Bun is chosen by its key under runtime, so the key is written even where +// nothing states a version: without it the control plane builds on Node. +func TestBunStarterChoosesBunUnderRuntime(t *testing.T) { + got := starterFor(typeNode, checkoutWith(t, map[string]string{ + "package.json": `{"scripts":{"build":"bun build ./index.ts --outdir dist","start":"bun dist/index.js"}}`, + "bun.lock": "{}", + })) + + if got.Type != typeNode || got.VersionKey != "bun" { + t.Errorf("Type = %q, VersionKey = %q, want nodejs on bun", got.Type, got.VersionKey) + } + + if got.Version != "1" || !strings.Contains(strings.Join(got.Notes, "\n"), "Bun 1") { + t.Errorf("Version = %q, Notes = %q, want Bun's major and a note saying why", got.Version, got.Notes) + } + + want := []string{ + "bun install --frozen-lockfile", + "bun run build", + "rm -rf node_modules && bun install --frozen-lockfile --production", + } + + if strings.Join(got.Build, " | ") != strings.Join(want, " | ") { + t.Errorf("Build = %q, want %q", got.Build, want) + } + + if !strings.Contains(renderStarter(got), "runtime:\n bun: '1'\n") { + t.Errorf("rendered without the key that chooses Bun:\n%s", renderStarter(got)) + } +} + +// node_modules travels in the release, so a Node build ends by leaving only +// what production needs — the step the control plane's own suggestion ends on. +func TestNodeStarterLeavesOnlyProductionDependencies(t *testing.T) { + got := starterFor(typeNode, checkoutWith(t, map[string]string{ + "package.json": `{"scripts":{"build":"next build","start":"next start"}}`, + })) + + want := []string{"npm ci", "npm run build", "npm prune --omit=dev"} + + if strings.Join(got.Build, " | ") != strings.Join(want, " | ") { + t.Errorf("Build = %q, want %q", got.Build, want) + } +} + +// The caches are the control plane's ManifestSuggestion::cachePaths(), and +// they are written under the build they belong to. +func TestStarterCachesWhatTheBuildInstalls(t *testing.T) { + cases := []struct { + kind string + files map[string]string + want []string + }{ + {typeDrupal, map[string]string{"composer.json": `{"require":{"drupal/core":"^11"}}`}, []string{"vendor", "web/core", "web/modules/contrib", "web/themes/contrib", "web/profiles/contrib", "web/libraries"}}, + {typeLaravel, map[string]string{"composer.json": `{}`}, []string{"vendor"}}, + {typeWordPress, map[string]string{"wp-config.php": " 0 && !strings.Contains(rendered, " cache:\n - "+c.want[0]+"\n") { + t.Errorf("no cache under the build:\n%s", rendered) + } + + if len(c.want) == 0 && strings.Contains(rendered, "cache:") { + t.Errorf("a cache with no build to keep it for:\n%s", rendered) + } + }) + } +} diff --git a/internal/cli/login.go b/internal/cli/login.go index 7ea49f1..4108d77 100644 --- a/internal/cli/login.go +++ b/internal/cli/login.go @@ -99,13 +99,10 @@ func runDeviceFlow( return nil, describeAuthFailure(err, client.BaseURL) } - // The complete URI first, because a code typed by hand is a code - // mistyped by hand. The plain one and the code are printed too: a + // The plain address, never one with the code in it: the code is typed + // on the page, from this terminal. Printed as well as opened, because a // browser on another machine is the whole reason this grant exists. - target := device.VerificationURIComplete - if target == "" { - target = device.VerificationURI - } + target := device.VerificationURI opened := false if !noBrowser && env.Interactive() { @@ -114,7 +111,7 @@ func runDeviceFlow( if opened { env.Printer.Say("Opened %s", target) - env.Printer.Say("Confirm the code shown there: %s", device.UserCode) + env.Printer.Say("Enter the code there: %s", device.UserCode) } else { env.Printer.Say("Open %s", device.VerificationURI) env.Printer.Say("and enter the code: %s", device.UserCode) diff --git a/internal/cli/operate.go b/internal/cli/operate.go new file mode 100644 index 0000000..d6cf1db --- /dev/null +++ b/internal/cli/operate.go @@ -0,0 +1,361 @@ +package cli + +import ( + "context" + "flag" + "fmt" + "strings" + "time" + + "github.com/vallic/vallic-cli/internal/api" + "github.com/vallic/vallic-cli/internal/output" +) + +// cdnCommand reads an environment's CDN settings and empties its cache. +// +// Read from the edge rather than from a copy, so what it prints is what the +// CDN is doing. Changing the settings stays in the console: the API takes +// every setting at once, as the console's form sends them, and a command that +// rewrote all of them to change one would be one copy away from undoing +// somebody else's change. +func cdnCommand() *Command { + return &Command{ + Name: "cdn", + Summary: "show an environment's CDN settings, and clear its cache", + Children: []*Command{ + { + Name: "show", + Summary: "show what the CDN caches, and for how long", + Usage: "cdn show []", + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + target, client, err := env.targetAndClient(ctx, first(args)) + if err != nil { + return err + } + + cdn, err := client.CDN(ctx, target.Environment.ID) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"cdn": cdn.Values}) + } + + values := cdn.Values + + return env.Printer.Print(output.Table{ + Columns: []string{"setting", "value"}, + Rows: [][]string{ + {"pages cached", yesNo(values.Dynamic)}, + {"browser cache, pages", cacheTime(values.BrowserCachePages)}, + {"browser cache, files", cacheTime(values.BrowserCacheFiles)}, + {"varies by", listOrNone(values.Vary)}, + {"cookies kept", listOrNone(values.Cookies)}, + {"headers kept", listOrNone(values.Headers)}, + {"countries refused", listOrNone(values.Blocked)}, + {"never cached", listOrNone(values.SkipPaths)}, + {"session cookies", listOrNone(values.SessionCookies)}, + }, + }, nil) + }, + }, + { + Name: "clear", + Aliases: []string{"purge"}, + Summary: "empty the CDN's cache for an environment", + Usage: "cdn clear []", + Long: `Asks the CDN to drop everything it holds for the environment. The edge +serves fresh copies from the site within a minute or so, and until then answers +from what it has. + +Nothing on the site changes: its own caches are its framework's, and drush cr +or its equivalent clears those.`, + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + target, client, err := env.targetAndClient(ctx, first(args)) + if err != nil { + return err + } + + said, err := client.ClearCDN(ctx, target.Environment.ID) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"cleared": true, "message": said.Message}) + } + + env.Printer.Good("%s", firstNonEmpty(said.Message, "The CDN cache is being cleared.")) + + return nil + }, + }, + }, + } +} + +// envMaintenanceCommand puts an environment behind its offline page. +func envMaintenanceCommand() *Command { + var ( + note string + yes bool + ) + + return &Command{ + Name: "maintenance", + Summary: "put an environment behind its offline page, or take it out", + Usage: "env maintenance on|off [] [--note ] [--yes]", + Long: `With on, visitors get the offline page from the next reconcile, within +the minute, and the site keeps running behind it: deploys, cron and a shell +all still work. off serves it again. + +--note is the one line the page shows, at most 200 characters; without it the +page keeps the note it had. + +Turning it on asks first, because it is visitors who see it. With no terminal +to ask on, pass --yes.`, + Flags: func(fs *flag.FlagSet) { + fs.StringVar(¬e, "note", "", "the line the offline page shows") + fs.BoolVar(&yes, "yes", false, "do not ask for confirmation") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return Usagef(nil, "say on or off") + } + + if err := ensureNoExtra(nil, args, 2); err != nil { + return err + } + + on, err := parseBool(args[0]) + if err != nil { + return Usagef(nil, "say on or off, not %q", args[0]) + } + + target, client, err := env.targetAndClient(ctx, firstOf(args[1:])) + if err != nil { + return err + } + + if on && !yes { + if !env.Interactive() { + return fmt.Errorf("this puts %s behind its offline page\n there is no terminal to confirm on, so pass --yes if you mean it", target.Named()) + } + + if !env.confirm(fmt.Sprintf("Put %s behind its offline page?", target.Named())) { + return fmt.Errorf("cancelled") + } + } + + var notePtr *string + if note != "" { + notePtr = ¬e + } + + result, err := client.SetMaintenance(ctx, target.Environment.ID, on, notePtr) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"maintenance": result}) + } + + env.Printer.Good("%s", result.Message) + + if result.Maintenance && result.Note != "" { + env.Printer.Say(" the page says: %s", result.Note) + } + + return nil + }, + } +} + +// metricsCommand summarises an environment's CPU, memory and disk. +// +// Upsun's metrics:all, as a summary rather than a chart: what it is using now, +// on average, and at worst, over the window the console's Resources tab shows. +func metricsCommand() *Command { + var window string + + return &Command{ + Name: "metrics", + Aliases: []string{"resources"}, + Summary: "show an environment's CPU, memory and disk use", + Usage: "metrics [] [--window 1h|24h|7d|30d|90d]", + Long: `The latest reading, the average and the peak over the window — the last +day unless --window says otherwise. Every reading is the average of its step, +a minute for an hour and longer for longer windows, so a peak shorter than a +step reads lower than it was. + +--format json prints every reading.`, + Flags: func(fs *flag.FlagSet) { + fs.StringVar(&window, "window", "24h", "how far back: 1h, 24h, 7d, 30d or 90d") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + target, client, err := env.targetAndClient(ctx, first(args)) + if err != nil { + return err + } + + resources, err := client.Resources(ctx, target.Environment.ID, window) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"resources": resources}) + } + + if len(resources.Points) == 0 { + env.Printer.Say("No readings for %s in the last %s.", target.Named(), resources.Window) + + return nil + } + + summary := summarise(resources.Points) + + env.Printer.Line("%s, the last %s, a reading every %s:", target.Named(), resources.Window, (time.Duration(resources.Step) * time.Second).Round(time.Second)) + env.Printer.Line("") + + return env.Printer.Print(output.Table{ + Columns: []string{"", "now", "average", "peak"}, + Rows: [][]string{ + {"cpu", percent(summary.cpu.latest), percent(summary.cpu.average), percent(summary.cpu.peak)}, + {"memory", humanBytes(int64(summary.memory.latest)), humanBytes(int64(summary.memory.average)), humanBytes(int64(summary.memory.peak))}, + {"disk", diskOrNone(summary.disk, summary.disk.latest), diskOrNone(summary.disk, summary.disk.average), diskOrNone(summary.disk, summary.disk.peak)}, + }, + }, nil) + }, + } +} + +// series is one measure over a window. +type series struct { + latest, average, peak float64 + count int +} + +func (s *series) add(value float64) { + s.latest = value + s.average += value + s.count++ + + if value > s.peak { + s.peak = value + } +} + +func (s *series) finish() { + if s.count > 0 { + s.average /= float64(s.count) + } +} + +type resourceSummary struct { + cpu, memory, disk series +} + +// summarise reduces the readings, oldest first, to now, average and peak. A +// step with no disk reading is left out of the disk's figures, not counted as +// an empty disk. +func summarise(points []api.ResourcePoint) resourceSummary { + var summary resourceSummary + + for _, point := range points { + summary.cpu.add(point.CPUPercent) + summary.memory.add(point.MemoryBytes) + + if point.DiskBytes != nil { + summary.disk.add(*point.DiskBytes) + } + } + + summary.cpu.finish() + summary.memory.finish() + summary.disk.finish() + + return summary +} + +func percent(value float64) string { + return fmt.Sprintf("%.0f%%", value) +} + +func diskOrNone(s series, value float64) string { + if s.count == 0 { + return "-" + } + + return humanBytes(int64(value)) +} + +// cacheTime is a browser cache time as somebody reads one, or what happens +// without one. +func cacheTime(seconds *int) string { + if seconds == nil { + return "the site's own headers" + } + + return humanSeconds(*seconds) +} + +// humanSeconds is a span as the console's own choices name it: 5 minutes, +// 1 hour, 30 days. +func humanSeconds(seconds int) string { + unit, size := "second", 1 + + switch { + case seconds >= 86400 && seconds%86400 == 0: + unit, size = "day", 86400 + case seconds >= 3600 && seconds%3600 == 0: + unit, size = "hour", 3600 + case seconds >= 60 && seconds%60 == 0: + unit, size = "minute", 60 + } + + n := seconds / size + if n != 1 { + unit += "s" + } + + return fmt.Sprintf("%d %s", n, unit) +} + +func listOrNone(values []string) string { + if len(values) == 0 { + return "none" + } + + return strings.Join(values, ", ") +} + +// targetAndClient resolves an environment and hands back the client that +// resolved it, for the commands that make one request about it. +func (e *Env) targetAndClient(ctx context.Context, positional string) (*Target, *api.Client, error) { + target, err := e.ResolveEnvironment(ctx, positional) + if err != nil { + return nil, nil, err + } + + client, err := e.Client() + if err != nil { + return nil, nil, err + } + + return target, client, nil +} diff --git a/internal/cli/operate_test.go b/internal/cli/operate_test.go new file mode 100644 index 0000000..185bd35 --- /dev/null +++ b/internal/cli/operate_test.go @@ -0,0 +1,105 @@ +package cli + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/vallic/vallic-cli/internal/api" +) + +// Upsun's spelling reaches the same command, so somebody coming from it can +// type what their fingers know. +func TestUpsunsSpellingIsReadAsTheWordsItStandsFor(t *testing.T) { + root := Root() + + cases := map[string][]string{ + "db:dump": {"db", "dump"}, + "environment:list": {"environment", "list"}, + "mount:upload": {"mount", "upload"}, + "ssh-key:add": {"ssh-key", "add"}, + "self:update": {"self-update"}, + "db:sql": {"sql"}, + "user:add": {"team", "invite"}, + "route:list": {"url", "--all"}, + } + + for typed, want := range cases { + got, ok := colonSpelling(root, typed) + if !ok || strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("colonSpelling(%q) = %q, %v; want %q", typed, got, ok, want) + } + } + + if got, ok := colonSpelling(root, "certificate:add"); ok { + t.Errorf("colonSpelling() made %q of a command with no equivalent", got) + } + + // Through the dispatcher: the colon form reaches the command, which + // refuses what it is given in its own words. + env, _, _ := followEnv(t) + err := Execute(context.Background(), root, env, []string{"db:dump", "--stdout", "--file", "x.sql"}) + + if err == nil || !strings.Contains(err.Error(), "--stdout writes no file") { + t.Errorf("db:dump reached %v, want db dump's own refusal", err) + } +} + +// Now, the average and the peak, with a step that has no disk reading left +// out of the disk's figures rather than counted as an empty disk. +func TestMetricsAreSummarisedWithoutInventingReadings(t *testing.T) { + disk := 2.0e9 + + summary := summarise([]api.ResourcePoint{ + {CPUPercent: 10, MemoryBytes: 100, DiskBytes: &disk}, + {CPUPercent: 50, MemoryBytes: 300}, + {CPUPercent: 30, MemoryBytes: 200}, + }) + + if summary.cpu.latest != 30 || summary.cpu.peak != 50 || summary.cpu.average != 30 { + t.Errorf("cpu = %+v", summary.cpu) + } + + if summary.memory.average != 200 || summary.memory.peak != 300 { + t.Errorf("memory = %+v", summary.memory) + } + + if summary.disk.count != 1 || summary.disk.latest != disk { + t.Errorf("disk = %+v, want the one reading there was", summary.disk) + } + + if diskOrNone(series{}, 0) != "-" { + t.Error("no disk readings printed as a size") + } +} + +// An empty --secret is none of them; no --secret is the control plane's guess. +func TestSecretNamesKnowWhetherTheyWereGiven(t *testing.T) { + var none namesFlag + none.Set("") + + if !none.given || len(none.names) != 0 { + t.Errorf("an empty --secret = %+v, want given and empty", none) + } + + var some namesFlag + some.Set("API_TOKEN, DB_PASSWORD") + + if strings.Join(some.names, " ") != "API_TOKEN DB_PASSWORD" { + t.Errorf("--secret = %q", some.names) + } +} + +// Putting a site behind its offline page is what visitors see, so it is not +// done without a yes — and with no terminal, the refusal says how to give one. +func TestMaintenanceAsksBeforeTheOfflinePage(t *testing.T) { + run := envMaintenanceCommand().Run + + for _, args := range [][]string{{}, {"maybe"}, {"on", "staging", "extra"}} { + var usage *UsageError + if err := run(context.Background(), nil, args); !errors.As(err, &usage) { + t.Errorf("%q = %v, want a usage error before anything is asked", args, err) + } + } +} diff --git a/internal/cli/people.go b/internal/cli/people.go new file mode 100644 index 0000000..ad4e04f --- /dev/null +++ b/internal/cli/people.go @@ -0,0 +1,442 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "strings" + "time" + + "github.com/vallic/vallic-cli/internal/api" + "github.com/vallic/vallic-cli/internal/output" +) + +// The people of the current team: who is in it, inviting more, their roles, +// taking them off. Upsun's user:list, user:add, user:update and user:delete, +// under the noun the platform has them on. +// +// The rules are the control plane's: a role is granted only by somebody who +// holds it, and anybody may take themselves off. Nothing here second-guesses +// them; a refusal arrives in its words. + +func teamPeopleCommand() *Command { + return &Command{ + Name: "people", + Summary: "list the team's members and its invitations still waiting", + Usage: "team people", + Long: `Everybody in the current team, their role and the projects it is narrowed +to, and the invitations nobody has accepted yet. The invitations are shown only +to somebody who may invite.`, + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 0); err != nil { + return err + } + + team, client, err := env.currentTeam(ctx) + if err != nil { + return err + } + + people, err := client.TeamPeople(ctx, team.ID) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(people) + } + + projects := projectLabels(people) + + members := output.Table{Columns: []string{"name", "email", "role", "projects"}} + for _, member := range people.Members { + name := member.User.Name + if member.IsSelf { + name += " (you)" + } + + email := "" + if member.User.Email != nil { + email = *member.User.Email + } + + members.Rows = append(members.Rows, []string{name, email, member.Role, projectsOf(member.Projects, projects)}) + } + + if err := env.Printer.Print(members, nil); err != nil { + return err + } + + if len(people.Invitations) == 0 { + return nil + } + + invitations := output.Table{Columns: []string{"invited", "role", "projects", "expires"}} + for _, invitation := range people.Invitations { + invitations.Rows = append(invitations.Rows, []string{ + invitation.Email, invitation.Role, projectsOf(invitation.Projects, projects), + time.Unix(invitation.Expires, 0).Format("2 Jan 2006"), + }) + } + + env.Printer.Line("") + + return env.Printer.Print(invitations, nil) + }, + } +} + +func teamInviteCommand() *Command { + var ( + role string + projects string + ) + + return &Command{ + Name: "invite", + Summary: "invite somebody to the team, and print the link to send them", + Usage: "team invite --role viewer|developer|admin|owner [--projects ]", + Long: `Makes an invitation and prints its link. No mail is sent: the link is +yours to pass on, and it is printed this once — only its digest is kept, so +there is no getting it back. Withdraw the invitation and make another if it is +lost. + +--projects narrows the role to those projects, by the name ` + "`vallic project list`" + ` +prints; without it the role reaches every project. Only some roles can be +narrowed, and only to a role you hold yourself can you invite.`, + Flags: func(fs *flag.FlagSet) { + fs.StringVar(&role, "role", "", "viewer, developer, admin or owner") + fs.StringVar(&projects, "projects", "", "the projects the role reaches, comma separated (default: every project)") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return Usagef(nil, "name the email address to invite") + } + + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + if role == "" { + return Usagef(nil, "--role is required: viewer, developer, admin or owner") + } + + team, client, err := env.currentTeam(ctx) + if err != nil { + return err + } + + ids, err := projectIDs(ctx, client, projects) + if err != nil { + return err + } + + invited, err := client.Invite(ctx, team.ID, args[0], role, ids) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(invited) + } + + env.Printer.Good("Invited %s to %s as %s.", invited.Invitation.Email, team.MachineName, invited.Invitation.Role) + env.Printer.Say(" no mail is sent; send them this link, which is not shown again:") + + // On stdout, alone: the one thing here worth capturing. + env.Printer.Line("%s", invited.Link) + + return nil + }, + } +} + +func teamRoleCommand() *Command { + var projects string + + return &Command{ + Name: "role", + Summary: "give a member another role", + Usage: "team role [--projects ]", + Long: `The member is named by email address or by name, as ` + "`vallic team people`" + ` +prints them. --projects narrows the role, as for team invite; without it the +role reaches every project.`, + Flags: func(fs *flag.FlagSet) { + fs.StringVar(&projects, "projects", "", "the projects the role reaches, comma separated (default: every project)") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) < 2 { + return Usagef(nil, "name the member and the role, as `vallic team role ana@example.com developer`") + } + + if err := ensureNoExtra(nil, args, 2); err != nil { + return err + } + + team, client, err := env.currentTeam(ctx) + if err != nil { + return err + } + + member, err := findMember(ctx, client, team, args[0]) + if err != nil { + return err + } + + ids, err := projectIDs(ctx, client, projects) + if err != nil { + return err + } + + changed, err := client.ChangeMembership(ctx, member.ID, args[1], ids) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(changed) + } + + env.Printer.Good("%s", firstNonEmpty(changed.Message, member.User.Name+" is now "+changed.Membership.Role+".")) + + return nil + }, + } +} + +func teamRemoveCommand() *Command { + var yes bool + + return &Command{ + Name: "remove", + Summary: "take a member off the team, or leave it", + Usage: "team remove [--yes]", + Long: `Takes the member off the current team; named as for team role. Their +keys stop opening the team's environments at the next reconcile. Naming +yourself leaves the team. + +It asks first. With no terminal to ask on, pass --yes.`, + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&yes, "yes", false, "do not ask for confirmation") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return Usagef(nil, "name the member to take off, by email or name") + } + + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + team, client, err := env.currentTeam(ctx) + if err != nil { + return err + } + + member, err := findMember(ctx, client, team, args[0]) + if err != nil { + return err + } + + question := fmt.Sprintf("Take %s off %s?", member.User.Name, team.MachineName) + if member.IsSelf { + question = fmt.Sprintf("Leave %s?", team.MachineName) + } + + if !yes { + if !env.Interactive() { + return fmt.Errorf("%s\n there is no terminal to confirm on, so pass --yes if you mean it", strings.TrimSuffix(question, "?")) + } + + if !env.confirm(question) { + return fmt.Errorf("cancelled") + } + } + + if err := client.RemoveMembership(ctx, member.ID); err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"removed": member}) + } + + if member.IsSelf { + env.Printer.Good("You have left %s.", team.MachineName) + } else { + env.Printer.Good("%s is no longer in %s.", member.User.Name, team.MachineName) + } + + return nil + }, + } +} + +func teamWithdrawCommand() *Command { + return &Command{ + Name: "withdraw", + Summary: "take back an invitation nobody has accepted", + Usage: "team withdraw ", + Long: `The invitation's link stops working. The address can be invited again.`, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return Usagef(nil, "name the email address the invitation went to") + } + + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + team, client, err := env.currentTeam(ctx) + if err != nil { + return err + } + + people, err := client.TeamPeople(ctx, team.ID) + if err != nil { + return err + } + + var found *api.TeamInvitation + for i := range people.Invitations { + if strings.EqualFold(people.Invitations[i].Email, args[0]) { + found = &people.Invitations[i] + break + } + } + + if found == nil { + return fmt.Errorf("no invitation to %s is waiting in %s\n `vallic team people` lists the ones that are", args[0], team.MachineName) + } + + if err := client.WithdrawInvitation(ctx, found.ID); err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"withdrawn": found}) + } + + env.Printer.Good("The invitation to %s is withdrawn; its link no longer works.", found.Email) + + return nil + }, + } +} + +// currentTeam is the team commands act in, as `team switch` recorded it, or +// the one team the credential reaches. +func (e *Env) currentTeam(ctx context.Context) (*api.Team, *api.Client, error) { + teams, current, err := e.teams(ctx) + if err != nil { + return nil, nil, err + } + + for i := range teams { + if teams[i].MachineName == current { + client, err := e.Client() + + return &teams[i], client, err + } + } + + if current != "" { + return nil, nil, fmt.Errorf("this credential does not reach %s, the team recorded as current\n choose another: vallic team switch ", current) + } + + return nil, nil, Usagef(nil, "this credential reaches several teams; choose one with `vallic team switch `, or pass --team") +} + +// findMember finds a member by email or name, which is how somebody knows +// them; the membership id is the API's. +func findMember(ctx context.Context, client *api.Client, team *api.Team, who string) (*api.TeamMember, error) { + people, err := client.TeamPeople(ctx, team.ID) + if err != nil { + return nil, err + } + + var matches []api.TeamMember + + for _, member := range people.Members { + email := "" + if member.User.Email != nil { + email = *member.User.Email + } + + if strings.EqualFold(email, who) || strings.EqualFold(member.User.Name, who) { + matches = append(matches, member) + } + } + + switch len(matches) { + case 1: + return &matches[0], nil + case 0: + return nil, fmt.Errorf("nobody called %s is in %s\n `vallic team people` lists who is", who, team.MachineName) + default: + return nil, errors.New("more than one member is called " + who + "; name them by email address") + } +} + +// projectIDs turns --projects, machine names or labels, into the ids the API +// takes. Empty is every project, which is nil. +func projectIDs(ctx context.Context, client *api.Client, names string) ([]int, error) { + if strings.TrimSpace(names) == "" { + return nil, nil + } + + projects, err := client.Projects(ctx) + if err != nil { + return nil, err + } + + ids := []int{} + + for _, name := range strings.Split(names, ",") { + name = strings.TrimSpace(name) + if name == "" { + continue + } + + found := false + for _, project := range projects { + if strings.EqualFold(project.MachineName, name) || strings.EqualFold(project.Label, name) { + ids = append(ids, project.ID) + found = true + + break + } + } + + if !found { + return nil, Usagef(nil, "no project called %q; `vallic project list` shows their names", name) + } + } + + return ids, nil +} + +// projectLabels are the team's projects by id, as the roster names them. +func projectLabels(people *api.TeamPeople) map[int]string { + labels := make(map[int]string, len(people.Projects)) + for _, project := range people.Projects { + labels[project.ID] = project.Label + } + + return labels +} + +// projectsOf is a role's projects as somebody reads them; none is every one. +func projectsOf(ids []int, labels map[int]string) string { + if len(ids) == 0 { + return "all" + } + + names := make([]string, 0, len(ids)) + for _, id := range ids { + names = append(names, firstNonEmpty(labels[id], fmt.Sprintf("#%d", id))) + } + + return strings.Join(names, ", ") +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 0401832..b41f9e0 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -30,6 +30,9 @@ to production.`, logoutCommand(), whoamiCommand(), teamCommand(), + // The keys every shell logs in with: the person's own, beside + // who they are signed in as. + sshKeyCommand(), // Starting in a checkout. Both run before anything on the // platform has to exist: init needs no credential at all, and @@ -50,6 +53,10 @@ to production.`, serviceCommand(), validateCommand(), domainCommand(), + // What the edge in front of a site holds, and the site's own + // use of its machines. + cdnCommand(), + metricsCommand(), // Keeping copies. backupCommand(), diff --git a/internal/cli/shell.go b/internal/cli/shell.go index efdeaf4..2ee8858 100644 --- a/internal/cli/shell.go +++ b/internal/cli/shell.go @@ -20,19 +20,27 @@ func identityFlag(fs *flag.FlagSet, into *string) { // target resolves an environment and builds a connection to it. func (e *Env) sshTarget(ctx context.Context, positional, identity string) (*vssh.Target, error) { + target, _, err := e.sshTargetFor(ctx, positional, identity) + + return target, err +} + +// sshTargetFor is sshTarget, keeping the project and environment it resolved +// for a command that names something after them. +func (e *Env) sshTargetFor(ctx context.Context, positional, identity string) (*vssh.Target, *Target, error) { resolved, err := e.ResolveEnvironment(ctx, positional) if err != nil { - return nil, err + return nil, nil, err } detail, err := e.Detail(ctx, resolved) if err != nil { - return nil, err + return nil, nil, err } target, err := vssh.New(detail, identity) if err != nil { - return nil, err + return nil, nil, err } if !target.HasKeys { @@ -42,10 +50,10 @@ func (e *Env) sshTarget(ctx context.Context, positional, identity string) (*vssh // fix is beats letting sshd answer with "Permission denied // (publickey)". cfg, _ := e.Config() - e.Printer.Warn("you have no SSH key on file; add one at %s", keysPage(cfg.API)) + e.Printer.Warn("you have no SSH key on file; add one with `vallic ssh-key add`, or at %s", keysPage(cfg.API)) } - return target, nil + return target, resolved, nil } // sshCommand opens a shell, or runs one command. @@ -191,8 +199,13 @@ func splitDrush(env *Env, args []string) (positional string, remote []string) { // dbCommand moves a database in or out. func dbCommand() *Command { - var identity string - var dryRun bool + var ( + identity string + dryRun bool + gzipped bool + file string + part dumpPart + ) flags := func(fs *flag.FlagSet) { identityFlag(fs, &identity) @@ -205,40 +218,87 @@ func dbCommand() *Command { Long: `Both directions stream, so size is not a limit and neither is a request timeout. + vallic db dump production a file, named for the project and environment + vallic db export production > f a stream, for a pipeline + vallic db import staging -f f a file or a stream, gzip or not + What runs on the far side is a verb, not a command: the agent expands db-export and db-import inside the container into the engine's own client — mysqldump or pg_dump, mysql or psql — with credentials that come from the container's environment and never appear on a command line. Which engine it is is decided in the container, not guessed from here.`, Children: []*Command{ + dbDumpCommand(), + dbSizeCommand(), { Name: "export", Summary: "stream a dump out, to stdout", - Usage: "db export [] > dump.sql", - Flags: flags, + Usage: "db export [] [--gzip] [--table ]... [--exclude-table ]... [--schema-only] > dump.sql", + Long: `The dump on standard output, for a redirect or a pipe. Refused onto a +terminal, where it would scroll past and be gone: ` + "`vallic db dump`" + ` writes a +file named for it instead. + +--gzip compresses it on the way out. --table, --exclude-table and +--schema-only take part of the database, as db dump's do.`, + Flags: func(fs *flag.FlagSet) { + flags(fs) + fs.BoolVar(&gzipped, "gzip", false, "compress the dump") + fs.BoolVar(&gzipped, "z", false, "compress the dump") + part.flags(fs) + }, Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + if err := part.check(); err != nil { + return err + } + target, err := env.sshTarget(ctx, first(args), identity) if err != nil { return err } - if target.DBExport == "" { - return fmt.Errorf("this environment runs no database") + verb, err := part.verb(target) + if err != nil { + return err } - // To stderr, because stdout is the dump. A progress line - // in the middle of a redirect would corrupt the file. - env.Printer.Say("Streaming %s out of %s…", "the database", target.Host) + target.Compress = true - return runOrPrint(env, target.Verb(target.DBExport, false), dryRun) + return streamDump(env, target.Verb(verb, false), gzipped, dryRun, target.Host) }, }, { Name: "import", - Summary: "stream a dump in, from stdin", - Usage: "db import [] < dump.sql", - Flags: flags, + Summary: "stream a dump in, from a file or stdin", + Usage: "db import [] [--file ] < dump.sql", + Long: `Reads the dump from --file (-f), or from standard input. A gzip dump is +decompressed on the way in, whichever it came from, so what db dump --gzip +wrote goes straight back: + + vallic db import staging -f acme--production--dump.sql.gz + vallic db import staging < production.sql + +It is read into the environment's database as it is, over what is there.`, + Flags: func(fs *flag.FlagSet) { + flags(fs) + fs.StringVar(&file, "file", "", "the dump to read, instead of standard input") + fs.StringVar(&file, "f", "", "the dump to read") + }, Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + if file == "" && env.stdinIsTerminal() { + // stdin is the terminal, which means no redirect was + // given and this would sit waiting for somebody to + // type a database. Better to say so than to hang. + return Usagef(nil, "nothing to import: name a dump with --file, or redirect one in, as `vallic db import < dump.sql`") + } + target, err := env.sshTarget(ctx, first(args), identity) if err != nil { return err @@ -248,39 +308,77 @@ is is decided in the container, not guessed from here.`, return fmt.Errorf("this environment runs no database") } - if env.Interactive() { - // stdin is the terminal, which means no redirect was - // given and this would sit waiting for somebody to - // type a database. Better to say so than to hang. - return Usagef(nil, "nothing to import: redirect a dump into this command, as `vallic db import < dump.sql`") + process := target.Verb(target.DBImport, false) + + if dryRun { + // By the name, since the dump is not opened for a dry + // run: the bytes decide on a real one. + switch { + case strings.HasSuffix(file, ".gz"): + env.Printer.Line("gunzip -c %s | %s", vssh.Quote(file), vssh.Describe(process)) + case file != "": + env.Printer.Line("%s < %s", vssh.Describe(process), vssh.Quote(file)) + default: + env.Printer.Line("%s", vssh.Describe(process)) + } + + return nil + } + + input, described, closer, err := openDump(env, file) + if err != nil { + return err } + defer closer() + + process.Stdin = input + + env.Printer.Say("Reading %s into the database on %s…", described, target.Host) - return runOrPrint(env, target.Verb(target.DBImport, false), dryRun) + return runOrPrint(env, process, false) }, }, }, } } -// sqlCommand is a database prompt. +// sqlCommand is a database prompt, or one query. func sqlCommand() *Command { var identity string var dryRun bool return &Command{ Name: "sql", - Summary: "open a database prompt in an environment", - Usage: "sql []", + Summary: "open a database prompt in an environment, or run a query", + Usage: "sql [] [-- ]", Long: `A prompt inside the container, not a local port. What you get is the engine's own client, running beside the site, reading its credentials from the container's environment. For a desktop client on a local -port instead, see vallic tunnel.`, +port instead, see vallic tunnel. + +After -- it runs that query and prints what the client answers, and with input +redirected it runs what it reads: + + vallic sql production -- 'SELECT COUNT(*) FROM users' + vallic sql staging < fix.sql`, Flags: func(fs *flag.FlagSet) { identityFlag(fs, &identity) fs.BoolVar(&dryRun, "dry-run", false, "print the ssh command instead of running it") }, Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) > 1 { + // Most often a query typed without the separator, which is + // the one thing that tells it from an environment's name. + return Usagef(nil, "unexpected argument %q; a query goes after --, as `vallic sql %s -- 'SELECT 1'`", args[1], args[0]) + } + + query, hasQuery := env.Passthrough() + + if hasQuery && len(query) == 0 { + return Usagef(nil, "nothing after --: name the query, as `vallic sql -- 'SELECT 1'`") + } + target, err := env.sshTarget(ctx, first(args), identity) if err != nil { return err @@ -294,10 +392,34 @@ port instead, see vallic tunnel.`, return fmt.Errorf("this environment runs no database") } - // A terminal, because this is a prompt: the forced command asks - // sshd for a pty and that is what makes the client interactive - // rather than something that reads stdin and exits. - return runOrPrint(env, target.Verb(target.DBCLI, true), dryRun) + switch { + case hasQuery: + // The verb is a fixed word and takes no arguments, so the + // query goes in the way a file would: on the client's input, + // with no terminal, where both clients run what they read and + // print the answer. + process := target.Verb(target.DBCLI, false) + joined := strings.Join(query, " ") + + if dryRun { + env.Printer.Line("printf '%%s\\n' %s | %s", vssh.Quote(joined), vssh.Describe(process)) + + return nil + } + + process.Stdin = strings.NewReader(joined + "\n") + + return runOrPrint(env, process, false) + case !env.stdinIsTerminal(): + // Input redirected: the same, from whatever was given. + return runOrPrint(env, target.Verb(target.DBCLI, false), dryRun) + default: + // A terminal, because this is a prompt: the forced command + // asks sshd for a pty and that is what makes the client + // interactive rather than something that reads stdin and + // exits. + return runOrPrint(env, target.Verb(target.DBCLI, true), dryRun) + } }, } } diff --git a/internal/cli/sshkey.go b/internal/cli/sshkey.go new file mode 100644 index 0000000..8c38775 --- /dev/null +++ b/internal/cli/sshkey.go @@ -0,0 +1,282 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/vallic/vallic-cli/internal/output" +) + +// sshKeyCommand manages the public keys on the signed-in person's account. +// +// Every shell, dump and file copy stands on one of these, and the first thing +// a new developer met used to be "you have no SSH key on file" and a page to +// find in the console. Upsun's `ssh-key:add` puts the key there from the +// terminal the person is already in, and so does this. +func sshKeyCommand() *Command { + return &Command{ + Name: "ssh-key", + Summary: "list, add and remove the SSH keys on your account", + Long: `The keys every shell, database dump and file copy logs in with. They are +yours rather than a team's: one key opens every environment your roles reach. + +They are managed with the credential vallic login stores. A personal access +token cannot reach them: a token reaches only what CI needs.`, + Children: []*Command{ + sshKeyListCommand(), + sshKeyAddCommand(), + sshKeyDeleteCommand(), + }, + } +} + +func sshKeyListCommand() *Command { + return &Command{ + Name: "list", + Summary: "list the keys on your account", + Usage: "ssh-key list", + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 0); err != nil { + return err + } + + client, err := env.Client() + if err != nil { + return err + } + + keys, err := client.SSHKeys(ctx) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"ssh_keys": keys}) + } + + table := output.Table{ + Columns: []string{"id", "label", "type", "fingerprint", "last used"}, + Empty: "No SSH keys on your account. Add one: vallic ssh-key add", + } + + for _, key := range keys { + used := "never" + if key.LastUsed != nil { + used = *key.LastUsed + } + + table.Rows = append(table.Rows, []string{fmt.Sprint(key.ID), key.Label, key.Type, key.Fingerprint, used}) + } + + return env.Printer.Print(table, nil) + }, + } +} + +func sshKeyAddCommand() *Command { + var label string + + return &Command{ + Name: "add", + Summary: "put a public key on your account", + Usage: "ssh-key add [] [--label ]", + Long: `Reads a public key and adds it to your account. With no file it takes the +one VALLIC_SSH_KEY names, or the first of ~/.ssh/id_ed25519.pub, +id_ecdsa.pub and id_rsa.pub — the keys ssh offers without being told. + +Naming the private key is fine: its .pub beside it is what is read, and the +private key never is. + +The label is the key's own comment where it has one, which ssh-keygen sets to +user@host. The fingerprint is printed back: compare it with + + ssh-keygen -lf ~/.ssh/id_ed25519.pub`, + Flags: func(fs *flag.FlagSet) { + fs.StringVar(&label, "label", "", "what to call the key on your account (default: its comment)") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + path, err := publicKeyPath(first(args)) + if err != nil { + return err + } + + key, comment, err := readPublicKey(path) + if err != nil { + return err + } + + if label == "" { + label = comment + } + + if label == "" { + label, _ = os.Hostname() + } + + client, err := env.Client() + if err != nil { + return err + } + + added, err := client.AddSSHKey(ctx, label, key) + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"ssh_key": added.Key}) + } + + env.Printer.Good("%s", firstNonEmpty(added.Message, "Key "+added.Key.Label+" added.")) + env.Printer.Say(" from %s", path) + + return nil + }, + } +} + +func sshKeyDeleteCommand() *Command { + return &Command{ + Name: "delete", + Aliases: []string{"remove"}, + Summary: "take a key off your account", + Usage: "ssh-key delete ", + Long: `Takes the key off your account, by the id ` + "`vallic ssh-key list`" + ` prints. +Nothing logs in with it after that; a session already open stays open.`, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return Usagef(nil, "name a key by the id `vallic ssh-key list` prints") + } + + if err := ensureNoExtra(nil, args, 1); err != nil { + return err + } + + id, err := atoi(args[0]) + if err != nil { + return Usagef(nil, "%q is not a key id; `vallic ssh-key list` prints them", args[0]) + } + + client, err := env.Client() + if err != nil { + return err + } + + if err := client.RemoveSSHKey(ctx, id); err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"deleted": map[string]any{"id": id}}) + } + + env.Printer.Good("Key %d removed.", id) + + return nil + }, + } +} + +// The public keys ssh offers without being told, in the order it tries them. +var defaultPublicKeys = []string{"id_ed25519.pub", "id_ecdsa.pub", "id_rsa.pub"} + +// publicKeyPath settles which file to read. +// +// A private key named is read through its .pub, which is the mistake most +// worth making harmless: the private half must never leave the machine, and +// it is what VALLIC_SSH_KEY and --identity name everywhere else. +func publicKeyPath(named string) (string, error) { + if named == "" { + named = os.Getenv("VALLIC_SSH_KEY") + } + + if named != "" { + named = expandHome(named) + + if !strings.HasSuffix(named, ".pub") { + if _, err := os.Stat(named + ".pub"); err == nil { + return named + ".pub", nil + } + } + + return named, nil + } + + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("cannot tell where ~/.ssh is: %w\n name the key: vallic ssh-key add path/to/key.pub", err) + } + + for _, name := range defaultPublicKeys { + path := filepath.Join(home, ".ssh", name) + + if _, err := os.Stat(path); err == nil { + return path, nil + } + } + + return "", errors.New("no public key in ~/.ssh\n make one: ssh-keygen -t ed25519\n then: vallic ssh-key add") +} + +// readPublicKey reads a public key file, and refuses anything else. +// +// The control plane checks the key properly; this checks only that the file +// is the right half, so a private key is refused here, before any of it is +// sent anywhere. +func readPublicKey(path string) (key, comment string, err error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", "", fmt.Errorf("cannot read %s: %w", path, err) + } + + text := strings.TrimSpace(string(raw)) + + if strings.Contains(text, "PRIVATE KEY") { + return "", "", fmt.Errorf("%s is a private key, which never leaves this machine\n name its .pub instead", path) + } + + fields := strings.Fields(text) + + if len(fields) < 2 || !(strings.HasPrefix(fields[0], "ssh-") || strings.HasPrefix(fields[0], "ecdsa-") || strings.HasPrefix(fields[0], "sk-")) { + return "", "", fmt.Errorf("%s does not hold an OpenSSH public key, which starts ssh-ed25519, ssh-rsa or ecdsa-", path) + } + + if len(fields) > 2 { + comment = strings.Join(fields[2:], " ") + } + + return strings.Join(fields[:2], " ") + optionalComment(comment), comment, nil +} + +func optionalComment(comment string) string { + if comment == "" { + return "" + } + + return " " + comment +} + +// expandHome turns a leading ~/ into the home directory, for a path that +// reached here without a shell to expand it: from VALLIC_SSH_KEY, or quoted. +func expandHome(path string) string { + rest, found := strings.CutPrefix(path, "~/") + if !found { + return path + } + + home, err := os.UserHomeDir() + if err != nil { + return path + } + + return filepath.Join(home, rest) +} diff --git a/internal/cli/sshkey_test.go b/internal/cli/sshkey_test.go new file mode 100644 index 0000000..90217ee --- /dev/null +++ b/internal/cli/sshkey_test.go @@ -0,0 +1,71 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/vallic/vallic-cli/internal/api" +) + +// The private half never leaves the machine. Named, it is read through the +// .pub beside it; read by mistake, it is refused before anything is sent. +func TestOnlyThePublicHalfOfAKeyIsRead(t *testing.T) { + dir := t.TempDir() + private := filepath.Join(dir, "vallic") + + if err := os.WriteFile(private, []byte("-----BEGIN OPENSSH PRIVATE KEY-----\nb3Blbn...\n-----END OPENSSH PRIVATE KEY-----\n"), 0o600); err != nil { + t.Fatal(err) + } + + if _, _, err := readPublicKey(private); err == nil || !strings.Contains(err.Error(), "private key") { + t.Errorf("readPublicKey(private) = %v, want a refusal", err) + } + + if err := os.WriteFile(private+".pub", []byte("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExample valentino@laptop\n"), 0o644); err != nil { + t.Fatal(err) + } + + path, err := publicKeyPath(private) + if err != nil || path != private+".pub" { + t.Fatalf("publicKeyPath(private) = %q, %v, want the .pub beside it", path, err) + } + + key, comment, err := readPublicKey(path) + if err != nil { + t.Fatalf("readPublicKey() = %v", err) + } + + if key != "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExample valentino@laptop" { + t.Errorf("key = %q", key) + } + + // The label a key is given when nobody names one. + if comment != "valentino@laptop" { + t.Errorf("comment = %q, want the key's own", comment) + } + + notAKey := filepath.Join(dir, "notes.txt") + if err := os.WriteFile(notAKey, []byte("remember the milk\n"), 0o644); err != nil { + t.Fatal(err) + } + + if _, _, err := readPublicKey(notAKey); err == nil { + t.Error("readPublicKey() accepted a file that holds no key") + } +} + +func TestTheSSHKeyAnswerDecodesAsTheControlPlaneSendsIt(t *testing.T) { + var answer api.SSHKeyAdded + body := `{"ssh_key":{"id":7,"label":"laptop","type":"ssh-ed25519","bits":null,"fingerprint":"SHA256:abc","comment":"v@laptop","last_used":null},"message":"Key laptop added."}` + + if err := json.Unmarshal([]byte(body), &answer); err != nil { + t.Fatal(err) + } + + if answer.Key.ID != 7 || answer.Key.Fingerprint != "SHA256:abc" || answer.Key.LastUsed != nil || answer.Message == "" { + t.Errorf("decoded as %+v", answer) + } +} diff --git a/internal/cli/team.go b/internal/cli/team.go index ce51b0a..191ea5c 100644 --- a/internal/cli/team.go +++ b/internal/cli/team.go @@ -24,10 +24,15 @@ import ( func teamCommand() *Command { return &Command{ Name: "team", - Summary: "show which teams this credential reaches", + Summary: "the teams this credential reaches, and their people", Children: []*Command{ teamListCommand(), teamSwitchCommand(), + teamPeopleCommand(), + teamInviteCommand(), + teamRoleCommand(), + teamRemoveCommand(), + teamWithdrawCommand(), }, } } diff --git a/internal/cli/variable.go b/internal/cli/variable.go index 90ac70b..a9d4f90 100644 --- a/internal/cli/variable.go +++ b/internal/cli/variable.go @@ -5,6 +5,9 @@ import ( "flag" "fmt" "io" + "os" + "slices" + "strings" "github.com/vallic/vallic-cli/internal/api" "github.com/vallic/vallic-cli/internal/output" @@ -45,6 +48,7 @@ environment that does not override it.`, varGetCommand(), varSetCommand(), varDeleteCommand(), + varImportCommand(), varApplyCommand(), }, } @@ -637,3 +641,191 @@ func takesEffect(answer string) string { return answer } + +// varImportCommand adds every variable in a .env, all or none. +// +// The console's bulk paste, from a file: the same parser reads it, the same +// guess marks the names that read like credentials as secret, and a name +// already defined refuses the whole import rather than half of it. +func varImportCommand() *Command { + var ( + scope string + secrets namesFlag + dryRun bool + ) + + return &Command{ + Name: "import", + Summary: "add every variable in a .env file, all or none", + Usage: "var import [] [--scope project] [--secret ] [--dry-run]", + Long: `Reads a .env — NAME=value lines, quoted or not, comments and export +allowed — and adds each one. It adds and never replaces: a name already set +here refuses the import, and nothing from the file is added. + +Names that read like credentials — PASSWORD, TOKEN, SECRET, KEY and the like — +are stored as secrets. --secret names exactly which ones are instead, and +--secret '' makes none of them secret. + +--dry-run prints what the file would add, and which as secrets, and adds +nothing. A file of - is read from standard input. + +As with var set, a running site sees them after the next deploy, or now with +var apply.`, + Flags: func(fs *flag.FlagSet) { + scopeFlag(&scope)(fs) + fs.Var(&secrets, "secret", "exactly the names to store as secrets, comma separated") + fs.BoolVar(&dryRun, "dry-run", false, "print what would be added, and add nothing") + }, + Run: func(ctx context.Context, env *Env, args []string) error { + if len(args) == 0 { + return Usagef(nil, "name a .env file, or - for standard input") + } + + chosen, err := variableScope(scope) + if err != nil { + return err + } + + taken := 2 + if chosen == scopeProject { + taken = 1 + } + + if err := ensureNoExtra(nil, args, taken); err != nil { + return err + } + + pasted, err := readDotEnv(env, args[0]) + if err != nil { + return err + } + + client, err := env.Client() + if err != nil { + return err + } + + if dryRun { + parsed, err := client.ParseVariables(ctx, pasted) + if err != nil { + return err + } + + if secrets.given { + for i := range parsed.Variables { + parsed.Variables[i].Secret = slices.Contains(secrets.names, parsed.Variables[i].Name) + } + } + + if env.Printer.Structured() { + return env.Printer.Value(map[string]any{"variables": parsed.Variables}) + } + + table := output.Table{ + Columns: []string{"name", "secret", "value"}, + Empty: "The file holds no variables.", + } + + for _, variable := range parsed.Variables { + value := variable.Value + if variable.Secret { + // Not printed: the point of a dry run is to check the + // file, and a secret on a screen is a secret shared. + value = "(hidden)" + } + + table.Rows = append(table.Rows, []string{variable.Name, yesNo(variable.Secret), value}) + } + + return env.Printer.Print(table, nil) + } + + var secret []string + if secrets.given { + secret = append([]string{}, secrets.names...) + } + + var imported *api.VariablesImported + + if chosen == scopeProject { + project, resolveErr := env.ResolveProject(ctx) + if resolveErr != nil { + return resolveErr + } + + imported, err = client.ImportProjectVariables(ctx, project.ID, pasted, secret) + } else { + target, resolveErr := env.ResolveEnvironment(ctx, firstOf(args[1:])) + if resolveErr != nil { + return resolveErr + } + + imported, err = client.ImportEnvironmentVariables(ctx, target.Environment.ID, pasted, secret) + } + + if err != nil { + return err + } + + if env.Printer.Structured() { + return env.Printer.Value(imported) + } + + env.Printer.Good("%s", firstNonEmpty(imported.Message, fmt.Sprintf("Added %d variables.", imported.Imported))) + env.Printer.Say(" they take effect at the next deploy, or now with: vallic var apply") + + return nil + }, + } +} + +// readDotEnv reads the file to import, or standard input for -. +func readDotEnv(env *Env, path string) (string, error) { + var ( + raw []byte + err error + ) + + if path == "-" { + if env.stdinIsTerminal() { + return "", Usagef(nil, "nothing to read: redirect a .env in, as `vallic var import - < .env`") + } + + raw, err = io.ReadAll(env.In) + } else { + raw, err = os.ReadFile(path) + } + + if err != nil { + return "", fmt.Errorf("cannot read %s: %w", path, err) + } + + if strings.TrimSpace(string(raw)) == "" { + return "", fmt.Errorf("%s is empty", path) + } + + return string(raw), nil +} + +// namesFlag is a comma-separated list that knows whether it was given at all, +// so an empty --secret, none of them, is told from no flag, the guess. +type namesFlag struct { + names []string + given bool +} + +func (n *namesFlag) String() string { + return strings.Join(n.names, ",") +} + +func (n *namesFlag) Set(value string) error { + n.given = true + + for _, name := range strings.Split(value, ",") { + if name = strings.TrimSpace(name); name != "" && !slices.Contains(n.names, name) { + n.names = append(n.names, name) + } + } + + return nil +} diff --git a/internal/resolve/resolve.go b/internal/resolve/resolve.go index 9f6cbc7..66b150f 100644 --- a/internal/resolve/resolve.go +++ b/internal/resolve/resolve.go @@ -408,7 +408,7 @@ func git(dir string, args ...string) string { // projectNames is the candidate list, each name with the label beside it. // // The name alone is what this printed and it is not enough to choose from. A -// machine name is generated — `brogxu30`, `egmcbg18` — so a list of five of +// machine name is generated — `abcdef12`, `ghijkl34` — so a list of five of // them asks somebody to pick their own project out of five strings that mean // nothing, and the answer is on the same response: the label they gave it. // diff --git a/internal/resolve/resolve_test.go b/internal/resolve/resolve_test.go index 19ef398..b105a3e 100644 --- a/internal/resolve/resolve_test.go +++ b/internal/resolve/resolve_test.go @@ -341,16 +341,16 @@ func (f *fakeClient) Environments(context.Context) ([]api.Environment, error) { // A project is offered by its label as well as its machine name. // // The name alone is what this printed, and a machine name is generated: -// `brogxu30`, `egmcbg18`. A list of five of them asks somebody to pick their own +// `abcdef12`, `ghijkl34`. A list of five of them asks somebody to pick their own // project out of five strings that mean nothing, when the label they gave it is // on the same response the candidates were built from. func TestProjectCandidatesCarryTheirLabels(t *testing.T) { got := projectNames([]api.Project{ - {MachineName: "brogxu30", Label: "Auctioneer"}, - {MachineName: "egmcbg18", Label: "Forum.tm"}, + {MachineName: "abcdef12", Label: "Blog"}, + {MachineName: "ghijkl34", Label: "Shop"}, }) - want := []string{"brogxu30 (Auctioneer)", "egmcbg18 (Forum.tm)"} + want := []string{"abcdef12 (Blog)", "ghijkl34 (Shop)"} if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("projectNames = %v, want %v", got, want) diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go index f18b45d..978128d 100644 --- a/internal/ssh/ssh.go +++ b/internal/ssh/ssh.go @@ -39,6 +39,12 @@ type Target struct { // Container is a container to open other than the application's: a // worker, by name. Container string + + // Compress asks ssh to compress the stream, for a verb whose output is a + // database dump: text that shrinks several times over, crossing the + // internet. The agent's verbs are fixed words, so `| gzip` cannot be + // added on the far side, and this is the compression that is left. + Compress bool } // jumpTarget is the machine a connection is carried on to, or empty. @@ -259,6 +265,10 @@ func (t *Target) Verb(verb string, tty bool) *exec.Cmd { args = append(args, "-T") } + if t.Compress { + args = append(args, "-C") + } + args = append(args, t.Destination(), t.named(verb)) return command("ssh", args...) @@ -318,6 +328,12 @@ func quoteAll(args []string) string { return strings.Join(quoted, " ") } +// Quote is quote, for a caller printing a line somebody can paste: a path +// beside a redirect in `--dry-run` output. +func Quote(arg string) string { + return quote(arg) +} + // quote wraps a single argument for the shell on the far side. // // Wrapped in single quotes, with an embedded single quote broken out into an diff --git a/internal/ssh/ssh_test.go b/internal/ssh/ssh_test.go index d7234a6..646ad71 100644 --- a/internal/ssh/ssh_test.go +++ b/internal/ssh/ssh_test.go @@ -124,6 +124,31 @@ func TestVerbStreamsWithoutATerminal(t *testing.T) { } } +// A dump is compressed on the wire only when asked, and never on the hop to +// the front: the ProxyCommand carries bytes ssh has already compressed. +func TestVerbCompressesOnlyWhenAsked(t *testing.T) { + plain := targetFor(2299) + + if contains(plain.Verb("db-export", false).Args, "-C") { + t.Error("compressed without being asked") + } + + compressed := targetFor(2299) + compressed.Compress = true + compressed.Machine = "web-2" + cmd := compressed.Verb("db-export", false) + + if !contains(cmd.Args, "-C") { + t.Errorf("args = %v, want -C", cmd.Args) + } + + for _, arg := range cmd.Args { + if strings.HasPrefix(arg, "ProxyCommand=") && strings.Contains(arg, " -C") { + t.Errorf("the hop compresses too: %q", arg) + } + } +} + // A prompt is the same verb mechanism with the pty asked for, because that is // the only difference between a client that reads a dump and one somebody // types into.