From 3071e6fe5dee03e50c165150275a379a4fa63414 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Tue, 17 Nov 2026 21:21:38 -0600 Subject: [PATCH 1/2] audio2vid_ltx2(): long audio-driven video as a chain of latent-carried LTX windows --- NAMESPACE | 2 + R/audio2vid_ltx2.R | 301 ++++++++++++++++++++++++++++ inst/tinytest/test_audio2vid_ltx2.R | 176 ++++++++++++++++ man/audio2vid_ltx2.Rd | 97 +++++++++ man/ltx2_windows.Rd | 29 +++ 5 files changed, 605 insertions(+) create mode 100644 R/audio2vid_ltx2.R create mode 100644 inst/tinytest/test_audio2vid_ltx2.R create mode 100644 man/audio2vid_ltx2.Rd create mode 100644 man/ltx2_windows.Rd diff --git a/NAMESPACE b/NAMESPACE index fc5353d..057a84c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,6 +1,7 @@ # tinyrox says don't edit this manually, but it can't stop you! export(animatediff_load_pipeline) +export(audio2vid_ltx2) export(auto_devices) export(bpe_tokenizer) export(clear_vram) @@ -121,6 +122,7 @@ export(load_unet_safetensors) export(load_unet_sdxl_safetensors) export(load_unet_sdxl_weights) export(load_unet_weights) +export(ltx2_windows) export(ltx23_ada_layer_norm_single) export(ltx23_adain_filter_latent) export(ltx23_antialias_act1d) diff --git a/R/audio2vid_ltx2.R b/R/audio2vid_ltx2.R new file mode 100644 index 0000000..a49ce3f --- /dev/null +++ b/R/audio2vid_ltx2.R @@ -0,0 +1,301 @@ +# audio2vid_ltx2.R +# Long audio-driven video: one narration of any length rendered as a +# chain of fixed LTX windows, each continuing the last from its tail +# latents, the way Lightricks' ltx-pipelines lays out long video. Window +# j + 1 starts `carry` frames before window j ends; those frames are +# carried as window j's last denoised latent frames (no decode, no +# re-encode), and every window hears the narration over its whole span, +# overlap included. The windows are written to `output_dir` as they +# finish, each with its tail latents and a record, so a run that stops +# resumes at the first window it does not have. + +#' Fixed windows over a narration +#' +#' Lays out windows of \code{window} frames that overlap by \code{carry} +#' frames and cover \code{frames}. Every window but the last is +#' \code{window} frames; the last is sized up to the end on the 8k+1 +#' grid and is never shorter than \code{carry + 8}. +#' +#' @param frames Frames to cover. +#' @param window Frames per window, 8k+1 (default 97). +#' @param carry Overlap in frames, 8k+1, at least 9 and below +#' \code{window} (default 25, four latent frames). +#' +#' @return Data frame with \code{index}, \code{start} (0-based frame) +#' and \code{frames}. +#' +#' @examples +#' ltx2_windows(300) +#' +#' @export +ltx2_windows <- function(frames, window = 97L, carry = 25L) { + window <- as.integer(window) + carry <- as.integer(carry) + if (!.on_8k1(window) || !.on_8k1(carry) || carry < 9L || + carry >= window) { + stop("window and carry must be 8k+1, with 9 <= carry < window", + call. = FALSE) + } + frames <- as.integer(ceiling(frames)) + if (length(frames) != 1L || is.na(frames) || frames < 1L) { + stop("no frames to cover", call. = FALSE) + } + starts <- 0L + while (starts[length(starts)] + window < frames) { + starts <- c(starts, starts[length(starts)] + window - carry) + } + n <- length(starts) + len <- rep(window, n) + rest <- frames - starts[n] + last <- 8L * as.integer(ceiling((rest - 1) / 8)) + 1L + len[n] <- min(window, max(last, if (n > 1L) carry + 8L else 9L)) + data.frame(index = seq_len(n), start = starts, frames = len) +} + +#' Long audio-driven video with LTX-2 +#' +#' Renders a narration of any length as a chain of LTX windows. The first +#' window starts from \code{image}; each later window is conditioned on +#' the previous window's last \code{carry} frames as latents, and every +#' window is driven by the narration over its own span. Each window is +#' written to \code{output_dir} as it finishes: \code{windowNN.mp4} (the +#' window's frames, its carried head included, with its audio), +#' \code{windowNN.latents.pt} (its tail latents) and \code{windowNN.json} +#' (its plan and prompt). Joining the windows (crossfading each carried +#' head over the previous window's tail) is left to the caller. +#' +#' A second call with the same arguments resumes: a window whose three +#' files exist and whose record matches the plan is kept, and the chain +#' continues from its saved latents. Once one window is rendered again, +#' every later window is too, since its conditioning changed. +#' +#' @param audio The narration: a file path (read with +#' \code{\link{ltx23_read_audio}}, which needs \code{av}) or a matrix +#' [2, samples] in [-1, 1] at 16 kHz. +#' @param image Start frame of the first window: a PNG/JPEG path or an +#' [H, W, 3] array in [0, 1]. +#' @param prompt Character. One prompt for every window, or one per +#' window. +#' @param pipeline An \code{ltx23_pipeline} (from +#' \code{\link{ltx23_load_pipeline}}) or a resident LTX handle (from +#' \code{\link{resident_load}}, active). +#' @param output_dir Directory for the windows; created if needed. +#' @param width,height Frame size, multiples of 32. +#' @param window,carry Window length and overlap in frames, 8k+1 (see +#' \code{\link{ltx2_windows}}). Ignored when \code{windows} is given, +#' except \code{carry}, which every plan shares. +#' @param windows Optional plan: a data frame with \code{start} and +#' \code{frames} (8k+1), each window starting \code{carry} frames before +#' the previous one ends, the first at 0. Default +#' \code{ltx2_windows(, window, carry)}. +#' @param frame_rate Frames per second. +#' @param seed Optional integer; window j renders with \code{seed + j - 1}. +#' @param resume Logical. Keep windows already on disk that match the plan. +#' @param on_window Optional function called with each window's record (a +#' list: \code{index}, \code{start}, \code{frames}, \code{file}, +#' \code{latents}, \code{rendered}, \code{seconds}) once it is on disk. +#' An error raised there stops the run after that window; the windows +#' written so far stay, so a later call resumes. +#' @param verbose Passed to \code{\link{txt2vid_ltx2}}. +#' @param generate Optional generator \code{function(prompt, ...)} in place +#' of the one \code{pipeline} implies; for tests and hosts. +#' @param ... Further arguments for \code{\link{txt2vid_ltx2}} +#' (\code{text_encoder}, \code{tokenizer}, \code{resident}, ...). +#' +#' @return Invisibly, a data frame with one row per window: +#' \code{index}, \code{start}, \code{frames}, \code{file}, +#' \code{latents}, \code{prompt} and \code{rendered} (FALSE when kept +#' from an earlier run). +#' +#' @export +audio2vid_ltx2 <- function(audio, image, prompt, pipeline = NULL, + output_dir, width = 768L, height = 512L, + window = 97L, carry = 25L, windows = NULL, + frame_rate = 24, seed = NULL, resume = TRUE, + on_window = NULL, verbose = "progress", + generate = NULL, ...) { + carry <- as.integer(carry) + width <- as.integer(width) + height <- as.integer(height) + if (width %% 32L != 0L || height %% 32L != 0L) { + stop("width and height must be multiples of 32", call. = FALSE) + } + gen <- generate %||% .audio2vid_generator(pipeline) + if (is.character(audio)) { + audio <- ltx23_read_audio(audio, sample_rate = 16000L) + } + if (!is.matrix(audio) || nrow(audio) != 2L || ncol(audio) < 1L) { + stop("audio must be a file or a [2, samples] matrix at 16 kHz", + call. = FALSE) + } + rate <- 16000 + bed_frames <- ceiling(ncol(audio) / rate * frame_rate - 1e-6) + if (is.null(windows)) { + windows <- ltx2_windows(bed_frames, window, carry) + } + windows <- .audio2vid_plan(windows, carry) + n <- nrow(windows) + if (!is.character(prompt) || anyNA(prompt) || + !length(prompt) %in% c(1L, n)) { + stop("prompt must be one string or one per window (", n, ")", + call. = FALSE) + } + prompts <- rep_len(prompt, n) + k <- (carry - 1L) %/% 8L + 1L + dir.create(output_dir, recursive = TRUE, showWarnings = FALSE) + + out <- data.frame(index = seq_len(n), start = windows$start, + frames = windows$frames, + file = file.path(output_dir, + sprintf("window%02d.mp4", seq_len(n))), + stringsAsFactors = FALSE) + out$latents <- sub("\\.mp4$", ".latents.pt", out$file) + out$prompt <- prompts + out$rendered <- FALSE + stale <- !isTRUE(resume) + tail <- NULL + for (j in seq_len(n)) { + t0 <- Sys.time() + rec <- list(start = out$start[j], frames = out$frames[j], + width = width, height = height, carry = carry, + frame_rate = frame_rate, prompt = prompts[j], + seed = if (is.null(seed)) NULL else as.integer(seed) + j - 1L, + samples = ncol(audio)) + json <- sub("\\.mp4$", ".json", out$file[j]) + if (!stale && .audio2vid_have(out$file[j], out$latents[j], json, rec)) { + tail <- NULL + } else { + stale <- TRUE + if (j > 1L && is.null(tail)) { + tail <- torch::torch_load(out$latents[j - 1L]) + } + args <- list(prompt = prompts[j], width = width, height = height, + num_frames = out$frames[j], frame_rate = frame_rate, + audio = .audio2vid_slice(audio, out$start[j], + out$frames[j], frame_rate, + rate), + filename = out$file[j], verbose = verbose) + if (!is.null(rec$seed)) { + args$seed <- rec$seed + } + if (j == 1L) { + args$image <- image + } else { + args$condition_latents <- tail + } + res <- do.call(gen, c(args, list(...))) + if (!file.exists(out$file[j])) { + stop("window ", j, " produced no file", call. = FALSE) + } + # A COPY, not the narrowed view: torch_save() of a view writes + # the start of its storage, so the saved tail would be the + # window's first latent frames and a resumed chain would + # continue from the wrong place. + tail <- ltx23_tail_latents(res, k)$to(device = "cpu")$clone() + rm(res) + torch::torch_save(tail, out$latents[j]) + writeLines(jsonlite::toJSON(rec, auto_unbox = TRUE, null = "null", + digits = NA), json) + out$rendered[j] <- TRUE + gc(verbose = FALSE) + } + if (is.function(on_window)) { + on_window(list(index = j, start = out$start[j], + frames = out$frames[j], file = out$file[j], + latents = out$latents[j], + rendered = out$rendered[j], + seconds = as.numeric(difftime(Sys.time(), t0, + units = "secs")))) + } + } + invisible(out) +} + +# 8k + 1, one value +.on_8k1 <- function(n) { + length(n) == 1L && !is.na(n) && n >= 1L && (n - 1L) %% 8L == 0L +} + +# The generator a pipeline implies: a resident handle's, or txt2vid_ltx2 +# on a loaded pipeline +.audio2vid_generator <- function(pipeline) { + if (inherits(pipeline, "diffuseR_resident")) { + if (!identical(pipeline$model, "ltx")) { + stop("the resident handle holds ", pipeline$model, ", not ltx", + call. = FALSE) + } + return(function(prompt, ...) resident_generate(pipeline, prompt, ...)) + } + if (inherits(pipeline, "ltx23_pipeline")) { + return(function(prompt, ...) txt2vid_ltx2(prompt, pipeline, ...)) + } + stop("pipeline must be an ltx23_pipeline or a resident ltx handle", + call. = FALSE) +} + +# A plan checked for a chain: integer start/frames, frames 8k+1 and +# longer than the carry, first at 0, each start `carry` frames before +# the previous window ends +.audio2vid_plan <- function(windows, carry) { + if (!is.data.frame(windows) || !nrow(windows) || + !all(c("start", "frames") %in% names(windows))) { + stop("windows must be a data frame with start and frames", + call. = FALSE) + } + if (!.on_8k1(carry) || carry < 9L) { + stop("carry must be 8k+1 and at least 9", call. = FALSE) + } + start <- as.integer(windows$start) + frames <- as.integer(windows$frames) + n <- length(start) + if (anyNA(start) || anyNA(frames) || + !all(vapply(frames, .on_8k1, logical(1)))) { + stop("every window's frames must be 8k+1", call. = FALSE) + } + if (start[1L] != 0L) { + stop("the first window must start at frame 0", call. = FALSE) + } + if (n > 1L) { + if (any(frames[-1L] <= carry)) { + stop("every window after the first must be longer than the carry (", + carry, ")", call. = FALSE) + } + want <- start[-n] + frames[-n] - carry + if (any(start[-1L] != want)) { + j <- which(start[-1L] != want)[1L] + 1L + stop("window ", j, " starts at ", start[j], "; the chain needs ", + want[j - 1L], " (the previous end less the carry)", + call. = FALSE) + } + } + data.frame(start = start, frames = frames) +} + +# The narration under one window, overlap included; the last window may +# reach past the end, and gets what is left +.audio2vid_slice <- function(audio, start, frames, frame_rate, rate) { + a <- as.integer(round(start / frame_rate * rate)) + 1L + b <- min(ncol(audio), as.integer(round((start + frames) / frame_rate * rate))) + if (a > ncol(audio)) { + stop("a window starts past the end of the audio", call. = FALSE) + } + audio[, a:b, drop = FALSE] +} + +# TRUE when a window's files exist and its record matches the plan +.audio2vid_have <- function(file, latents, json, rec) { + if (!file.exists(file) || !file.exists(latents) || !file.exists(json)) { + return(FALSE) + } + old <- tryCatch(jsonlite::fromJSON(json, simplifyVector = TRUE), + error = function(e) NULL) + if (is.null(old)) { + return(FALSE) + } + for (nm in names(rec)) { + if (!identical(as.character(old[[nm]]), as.character(rec[[nm]]))) { + return(FALSE) + } + } + TRUE +} diff --git a/inst/tinytest/test_audio2vid_ltx2.R b/inst/tinytest/test_audio2vid_ltx2.R new file mode 100644 index 0000000..c394d86 --- /dev/null +++ b/inst/tinytest/test_audio2vid_ltx2.R @@ -0,0 +1,176 @@ +# audio2vid_ltx2(): the window plan, the latent chain, the audio under +# each window, and resume. A fake generator stands in for LTX: it writes +# a placeholder file and returns packed latents whose values name the +# window and latent frame, so the carried tail can be read back. CPU only. + +if (!requireNamespace("torch", quietly = TRUE) || !torch::torch_is_installed()) { + exit_file("torch not fully installed") +} + +library(diffuseR) + +# --- the plan --------------------------------------------------------------- + +w <- ltx2_windows(300) +expect_equal(w$start, c(0L, 72L, 144L, 216L)) +expect_equal(w$frames, c(97L, 97L, 97L, 89L)) # 84 left, up to 89 +expect_true(all((w$frames - 1L) %% 8L == 0L)) +expect_true(w$start[4] + w$frames[4] >= 300) + +w <- ltx2_windows(150) +expect_equal(w$start, c(0L, 72L)) +expect_equal(w$frames, c(97L, 81L)) # 78 left, up to 81 on the grid + +expect_equal(nrow(ltx2_windows(50)), 1L) +expect_equal(ltx2_windows(50)$frames, 57L) +expect_equal(ltx2_windows(98)$frames, c(97L, 33L)) # never shorter than carry + 8 +expect_error(ltx2_windows(100, window = 96L), "8k\\+1") +expect_error(ltx2_windows(100, carry = 97L), "carry < window") +expect_error(ltx2_windows(0), "no frames") + +# --- a fake generator ------------------------------------------------------- + +calls <- list() +fake <- function(prompt, width, height, num_frames, frame_rate, audio, + filename, verbose, image = NULL, condition_latents = NULL, + seed = NULL, ...) { + j <- length(calls) + 1L + calls[[j]] <<- list(prompt = prompt, num_frames = num_frames, + samples = ncol(audio), image = image, + cond = if (is.null(condition_latents)) NULL else + as.numeric(condition_latents$flatten()), + cond_shape = if (is.null(condition_latents)) NULL else + as.integer(condition_latents$shape), + seed = seed, extra = list(...)) + writeLines("mp4", filename) + lf <- (num_frames - 1L) %/% 8L + 1L + # packed [1, lf * 1 * 1, 128]: every channel of latent frame f holds + # 1000 * call + f + v <- rep(1000 * j + seq_len(lf), times = 128L) # [1, f, c], column-major + list(latents = torch::torch_tensor(array(v, c(1L, lf, 128L)), + dtype = torch::torch_float32()), + latent_shape = c(lf, 1L, 1L)) +} + +rate <- 16000 +secs <- 300 / 24 # 300 frames of narration +audio <- matrix(0, 2L, as.integer(secs * rate)) +dir <- tempfile("a2v-") + +out <- audio2vid_ltx2(audio, image = "start.png", prompt = "a corn cob talks", + output_dir = dir, width = 64L, height = 64L, + generate = fake, text_encoder = "te") +expect_equal(nrow(out), 4L) +expect_true(all(out$rendered)) +expect_equal(length(calls), 4L) +expect_true(all(file.exists(out$file))) +expect_true(all(file.exists(out$latents))) +expect_true(all(file.exists(sub("\\.mp4$", ".json", out$file)))) + +# the first window starts from the image, the rest from latents +expect_equal(calls[[1]]$image, "start.png") +expect_null(calls[[1]]$cond) +expect_null(calls[[2]]$image) +# carry 25 -> 4 latent frames: the last four of window 1 (13 latent frames) +expect_equal(calls[[2]]$cond_shape, c(1L, 128L, 4L, 1L, 1L)) +expect_equal(unique(calls[[2]]$cond), 1000 + 10:13) +expect_equal(unique(calls[[4]]$cond), 3000 + 10:13) + +# every window hears its own span, overlap included +expect_equal(calls[[1]]$samples, round(97 / 24 * rate)) +expect_equal(calls[[2]]$samples, + round((72 + 97) / 24 * rate) - round(72 / 24 * rate)) +expect_equal(calls[[4]]$samples, ncol(audio) - round(216 / 24 * rate)) +expect_equal(vapply(calls, `[[`, 1L, "num_frames"), c(97L, 97L, 97L, 89L)) +expect_equal(calls[[3]]$extra$text_encoder, "te") +expect_null(calls[[1]]$seed) + +# --- resume ----------------------------------------------------------------- + +calls <- list() +out2 <- audio2vid_ltx2(audio, image = "start.png", prompt = "a corn cob talks", + output_dir = dir, width = 64L, height = 64L, + generate = fake) +expect_equal(length(calls), 0L) +expect_false(any(out2$rendered)) + +# window 3 lost: windows 1-2 kept, 3 renders from window 2's saved +# latents, and 4 renders again because its conditioning changed +unlink(out$latents[3]) +calls <- list() +out3 <- audio2vid_ltx2(audio, image = "start.png", prompt = "a corn cob talks", + output_dir = dir, width = 64L, height = 64L, + generate = fake) +expect_equal(out3$rendered, c(FALSE, FALSE, TRUE, TRUE)) +expect_equal(length(calls), 2L) +expect_equal(unique(calls[[1]]$cond), 2000 + 10:13) # window 2 of the first run +expect_equal(unique(calls[[2]]$cond), 1000 + 10:13) # this run's window 3 + +# a changed prompt for one window re-renders from there on +calls <- list() +out4 <- audio2vid_ltx2(audio, image = "start.png", + prompt = c("a corn cob talks", "a corn cob talks", + "a corn cob sings", "a corn cob talks"), + output_dir = dir, width = 64L, height = 64L, + generate = fake) +expect_equal(out4$rendered, c(FALSE, FALSE, TRUE, TRUE)) +expect_equal(calls[[1]]$prompt, "a corn cob sings") + +# resume = FALSE renders everything +calls <- list() +out5 <- audio2vid_ltx2(audio, image = "start.png", prompt = "x", + output_dir = dir, width = 64L, height = 64L, + resume = FALSE, generate = fake, seed = 7L) +expect_true(all(out5$rendered)) +expect_equal(vapply(calls, `[[`, 1L, "seed"), 7:10) + +# --- on_window: progress, and a stop that leaves resumable windows --------- + +seen <- integer(0) +dir2 <- tempfile("a2v-") +calls <- list() +res <- tryCatch( + audio2vid_ltx2(audio, image = "start.png", prompt = "x", output_dir = dir2, + width = 64L, height = 64L, generate = fake, + on_window = function(r) { + seen <<- c(seen, r$index) + if (r$index == 2L) stop("preempted") + }), + error = function(e) conditionMessage(e)) +expect_equal(res, "preempted") +expect_equal(seen, 1:2) +calls <- list() +out6 <- audio2vid_ltx2(audio, image = "start.png", prompt = "x", + output_dir = dir2, width = 64L, height = 64L, + generate = fake) +expect_equal(out6$rendered, c(FALSE, FALSE, TRUE, TRUE)) + +# --- a caller's plan -------------------------------------------------------- + +plan <- data.frame(start = c(0L, 120L), frames = c(145L, 113L)) +calls <- list() +out7 <- audio2vid_ltx2(matrix(0, 2L, as.integer(10 * rate)), image = "s.png", + prompt = c("a", "b"), output_dir = tempfile("a2v-"), + width = 64L, height = 64L, windows = plan, + generate = fake) +expect_equal(out7$frames, c(145L, 113L)) +expect_equal(vapply(calls, `[[`, "", "prompt"), c("a", "b")) + +# --- refusals --------------------------------------------------------------- + +d <- tempfile("a2v-") +expect_error(audio2vid_ltx2(audio, "s.png", "x", output_dir = d, + width = 60L, generate = fake), "multiples of 32") +expect_error(audio2vid_ltx2(audio, "s.png", c("a", "b"), output_dir = d, + generate = fake), "one per window") +expect_error(audio2vid_ltx2(matrix(0, 1L, 10L), "s.png", "x", output_dir = d, + generate = fake), "\\[2, samples\\]") +expect_error(audio2vid_ltx2(audio, "s.png", "x", output_dir = d, + windows = data.frame(start = c(0L, 80L), + frames = c(97L, 97L)), + generate = fake), "the chain needs 72") +expect_error(audio2vid_ltx2(audio, "s.png", "x", output_dir = d, + windows = data.frame(start = 0L, frames = 96L), + generate = fake), "8k\\+1") +expect_error(audio2vid_ltx2(audio, "s.png", "x", output_dir = d), + "ltx23_pipeline or a resident") diff --git a/man/audio2vid_ltx2.Rd b/man/audio2vid_ltx2.Rd new file mode 100644 index 0000000..afef5f5 --- /dev/null +++ b/man/audio2vid_ltx2.Rd @@ -0,0 +1,97 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{audio2vid_ltx2} +\alias{audio2vid_ltx2} +\title{Long audio-driven video with LTX-2} +\usage{ +audio2vid_ltx2( + audio, + image, + prompt, + pipeline = NULL, + output_dir, + width = 768L, + height = 512L, + window = 97L, + carry = 25L, + windows = NULL, + frame_rate = 24, + seed = NULL, + resume = TRUE, + on_window = NULL, + verbose = "progress", + generate = NULL, + ... +) +} +\arguments{ +\item{audio}{The narration: a file path (read with +\code{\link{ltx23_read_audio}}, which needs \code{av}) or a matrix +[2, samples] in [-1, 1] at 16 kHz.} + +\item{image}{Start frame of the first window: a PNG/JPEG path or an +[H, W, 3] array in [0, 1].} + +\item{prompt}{Character. One prompt for every window, or one per +window.} + +\item{pipeline}{An \code{ltx23_pipeline} (from +\code{\link{ltx23_load_pipeline}}) or a resident LTX handle (from +\code{\link{resident_load}}, active).} + +\item{output_dir}{Directory for the windows; created if needed.} + +\item{windows}{Optional plan: a data frame with \code{start} and +\code{frames} (8k+1), each window starting \code{carry} frames before +the previous one ends, the first at 0. Default +\code{ltx2_windows(, window, carry)}.} + +\item{frame_rate}{Frames per second.} + +\item{seed}{Optional integer; window j renders with \code{seed + j - 1}.} + +\item{resume}{Logical. Keep windows already on disk that match the plan.} + +\item{on_window}{Optional function called with each window's record (a +list: \code{index}, \code{start}, \code{frames}, \code{file}, +\code{latents}, \code{rendered}, \code{seconds}) once it is on disk. +An error raised there stops the run after that window; the windows +written so far stay, so a later call resumes.} + +\item{verbose}{Passed to \code{\link{txt2vid_ltx2}}.} + +\item{generate}{Optional generator \code{function(prompt, ...)} in place +of the one \code{pipeline} implies; for tests and hosts.} + +\item{...}{Further arguments for \code{\link{txt2vid_ltx2}} +(\code{text_encoder}, \code{tokenizer}, \code{resident}, ...).} + +\item{width,height}{Frame size, multiples of 32.} + +\item{window,carry}{Window length and overlap in frames, 8k+1 (see +\code{\link{ltx2_windows}}). Ignored when \code{windows} is given, +except \code{carry}, which every plan shares.} +} +\value{ +Invisibly, a data frame with one row per window: + \code{index}, \code{start}, \code{frames}, \code{file}, + \code{latents}, \code{prompt} and \code{rendered} (FALSE when kept + from an earlier run). +} +\description{ +Renders a narration of any length as a chain of LTX windows. The first +window starts from \code{image}; each later window is conditioned on +the previous window's last \code{carry} frames as latents, and every +window is driven by the narration over its own span. Each window is +written to \code{output_dir} as it finishes: \code{windowNN.mp4} (the +window's frames, its carried head included, with its audio), +\code{windowNN.latents.pt} (its tail latents) and \code{windowNN.json} +(its plan and prompt). Joining the windows (crossfading each carried +head over the previous window's tail) is left to the caller. +} +\details{ +A second call with the same arguments resumes: a window whose three +files exist and whose record matches the plan is kept, and the chain +continues from its saved latents. Once one window is rendered again, +every later window is too, since its conditioning changed. + +} diff --git a/man/ltx2_windows.Rd b/man/ltx2_windows.Rd new file mode 100644 index 0000000..953086f --- /dev/null +++ b/man/ltx2_windows.Rd @@ -0,0 +1,29 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx2_windows} +\alias{ltx2_windows} +\title{Fixed windows over a narration} +\usage{ +ltx2_windows(frames, window = 97L, carry = 25L) +} +\arguments{ +\item{frames}{Frames to cover.} + +\item{window}{Frames per window, 8k+1 (default 97).} + +\item{carry}{Overlap in frames, 8k+1, at least 9 and below +\code{window} (default 25, four latent frames).} +} +\value{ +Data frame with \code{index}, \code{start} (0-based frame) + and \code{frames}. +} +\description{ +Lays out windows of \code{window} frames that overlap by \code{carry} +frames and cover \code{frames}. Every window but the last is +\code{window} frames; the last is sized up to the end on the 8k+1 +grid and is never shorter than \code{carry + 8}. +} +\examples{ +ltx2_windows(300) + +} From 4fef3a907b6195b2d9270d69ddf7faba95ffe8be Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Tue, 17 Nov 2026 21:21:48 -0600 Subject: [PATCH 2/2] Bump version to 0.2.2.28 --- DESCRIPTION | 2 +- NEWS.md | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 8921d9b..b428435 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: diffuseR Title: Functional Interface to Diffusion Models in R -Version: 0.2.2.27 +Version: 0.2.2.28 Authors@R: c( person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"), comment = c(ORCID = "0009-0005-4248-604X")), diff --git a/NEWS.md b/NEWS.md index 1439409..8845cb3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,19 @@ +# diffuseR 0.2.2.28 + +* `audio2vid_ltx2(audio, image, prompt, pipeline, output_dir)` renders a + narration of any length as a chain of LTX windows: the first from the + start image, each later one conditioned on the previous window's last + `carry` frames as latents, every window driven by the narration over its + own span. Windows are written as they finish (`windowNN.mp4`, its tail + latents, its record), so a second call resumes at the first window it + lacks; `on_window` reports each one and may stop the run. One prompt or + one per window; a resident LTX handle or a loaded pipeline. The chain + that callers ran one request at a time (cornductor's timeline layout over + xtx.api) now runs in one call. +* `ltx2_windows(frames, window = 97, carry = 25)` is the fixed plan it uses + by default: `window`-frame windows overlapping by `carry`, the last sized + to the end on the 8k+1 grid. + # diffuseR 0.2.2.27 * `resident_generate(res, prompt, image = )` on a `flux3` handle runs the