[Brook S1] Added the Stream service protos and their generated code.

pyproject.toml

@@ -183,12 +183,19 @@ exclude = [
   'temporalio/api',
   'temporalio/bridge/proto',
   'temporalio/nexus/system/workflow_service',
+  'temporalio/streams/proto',
 ]
 
 [[tool.mypy.overrides]]
 module = "temporalio.nexus.system.workflow_service.*"
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+module = [
+  "temporalio.streams.proto.*",
+]
+ignore_errors = true
+
 [tool.pydocstyle]
 convention = "google"
 # https://github.com/PyCQA/pydocstyle/issues/363#issuecomment-625563088
@@ -213,6 +220,7 @@ intersphinx = [
 ]
 privacy = [
   "PRIVATE:temporalio.bridge",
+  "PRIVATE:temporalio.streams.proto",
   "PRIVATE:temporalio.types",
   "PRIVATE:temporalio.worker.workflow_sandbox.restrictions",
   "PRIVATE:temporalio.worker.workflow_sandbox.runner",
@@ -258,6 +266,7 @@ exclude = [
   "temporalio/api",
   "temporalio/bridge/proto",
   "temporalio/bridge/_visitor.py",
+  "temporalio/streams/proto",
   "tests/worker/workflow_sandbox/testmodules/proto",
 ]
 

scripts/_proto/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.10
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 ENV PATH="$PATH:/root/.local/bin"
 RUN uv tool install poethepoet
-VOLUME ["/api_new", "/bridge_new"]
+VOLUME ["/api_new", "/bridge_new", "/streams_new"]
 
 COPY ./ ./
 
@@ -13,4 +13,4 @@ RUN uv add "protobuf<4"
 RUN uv sync --all-extras
 RUN uv run scripts/gen_protos.py
 
-CMD ["sh", "-c", "cp -r ./temporalio/api/* /api_new && cp -r ./temporalio/bridge/proto/* /bridge_new"]
+CMD ["sh", "-c", "cp -r ./temporalio/api/* /api_new && cp -r ./temporalio/bridge/proto/* /bridge_new && cp -r ./temporalio/streams/proto/* /streams_new"]

scripts/_proto/temporal/api/enums/v1/stream.proto

@@ -0,0 +1,83 @@
+syntax = "proto3";
+
+package temporal.api.enums.v1;
+
+option go_package = "go.temporal.io/api/enums/v1;enums";
+option java_package = "io.temporal.api.enums.v1";
+option java_multiple_files = true;
+option java_outer_classname = "StreamProto";
+option ruby_package = "Temporalio::Api::Enums::V1";
+option csharp_namespace = "Temporalio.Api.Enums.V1";
+
+// Experimental. Might change in the future.
+// The state of a Stream, as a describe gives it.
+// (-- api-linter: core::0216::synonyms=disabled
+//     aip.dev/not-precedent: Named consistently with WorkflowExecutionStatus. --)
+enum StreamStatus {
+    STREAM_STATUS_UNSPECIFIED = 0;
+    STREAM_STATUS_OPEN = 1;
+    STREAM_STATUS_CLOSED = 2;
+    STREAM_STATUS_EXPIRED = 3;
+}
+
+// Experimental. Might change in the future.
+// Where a read starts when it has no from_offset.
+enum StreamReadStart {
+    STREAM_READ_START_UNSPECIFIED = 0;
+    STREAM_READ_START_BEGINNING = 1;
+    STREAM_READ_START_END = 2;
+}
+
+// Experimental. Might change in the future.
+// What a create does when its Stream exists. Unspecified is FAIL, as for a Workflow start.
+enum StreamIdConflictPolicy {
+    STREAM_ID_CONFLICT_POLICY_UNSPECIFIED = 0;
+    // The create fails with ALREADY_EXISTS.
+    STREAM_ID_CONFLICT_POLICY_FAIL = 1;
+    // The create gets the open Stream, which keeps its own Retention.
+    STREAM_ID_CONFLICT_POLICY_USE_EXISTING = 2;
+}
+
+// Experimental. Might change in the future.
+// Why a Stream call failed. Each reason has one gRPC status code. An SDK raises its own error
+// class for each reason that code can act on, and a base class for the others.
+enum StreamErrorReason {
+    STREAM_ERROR_REASON_UNSPECIFIED = 0;
+    // Nobody created the Stream, even after `create_wait`. NOT_FOUND.
+    STREAM_ERROR_REASON_NOT_FOUND = 1;
+    // The Stream's Retention or safety TTL passed. NOT_FOUND.
+    STREAM_ERROR_REASON_EXPIRED = 2;
+    // The Stream is closed. FAILED_PRECONDITION.
+    STREAM_ERROR_REASON_CLOSED = 3;
+    // The Producer's epoch is lower than the newest one, its Workflow Task was withdrawn, or a
+    // Workflow retry withdrew its run. FAILED_PRECONDITION.
+    STREAM_ERROR_REASON_FENCED = 4;
+    // The batch has a lower `sequence` than the last one of its epoch. FAILED_PRECONDITION.
+    STREAM_ERROR_REASON_SEQUENCE_TOO_LOW = 5;
+    // The batch has the `sequence` of the last one of its epoch, and other content.
+    // FAILED_PRECONDITION.
+    STREAM_ERROR_REASON_SEQ_DIVERGENT = 6;
+    // A create gives another owner than the Stream has, or the owner doesn't allow this close.
+    // FAILED_PRECONDITION.
+    STREAM_ERROR_REASON_NOT_OWNER = 7;
+    // A newer format version wrote the Stream. FAILED_PRECONDITION.
+    STREAM_ERROR_REASON_FORMAT_TOO_NEW = 8;
+    // The offset is past the newest offset + 1, for example after a failover lost the newest
+    // Records. A read from it works again once the Stream has Records up to it. OUT_OF_RANGE.
+    STREAM_ERROR_REASON_OFFSET_TOO_HIGH = 9;
+    // A Record or a batch is over the Store's limit. INVALID_ARGUMENT.
+    STREAM_ERROR_REASON_TOO_LARGE = 10;
+    // The request is malformed. INVALID_ARGUMENT.
+    STREAM_ERROR_REASON_INVALID = 11;
+    // The Store failed or did not answer. A retry can succeed. UNAVAILABLE.
+    STREAM_ERROR_REASON_STORAGE_UNAVAILABLE = 12;
+    // No writer has the id. The SDK's in-process writer only. NOT_FOUND.
+    STREAM_ERROR_REASON_UNKNOWN_WRITER = 13;
+    // The Store lacks a capability that the call needs. UNIMPLEMENTED.
+    STREAM_ERROR_REASON_UNSUPPORTED = 14;
+    // A create with the FAIL conflict policy is for a Stream that exists. ALREADY_EXISTS.
+    STREAM_ERROR_REASON_ALREADY_EXISTS = 15;
+    // The Store gave back data that isn't valid, such as a stored value with the wrong length. A
+    // retry reads the same data. DATA_LOSS.
+    STREAM_ERROR_REASON_STORAGE_CORRUPT = 16;
+}

scripts/_proto/temporal/api/stream/v1/message.proto

@@ -0,0 +1,153 @@
+syntax = "proto3";
+
+package temporal.api.stream.v1;
+
+option go_package = "go.temporal.io/api/stream/v1;stream";
+option java_package = "io.temporal.api.stream.v1";
+option java_multiple_files = true;
+option java_outer_classname = "MessageProto";
+option ruby_package = "Temporalio::Api::Stream::V1";
+option csharp_namespace = "Temporalio.Api.Stream.V1";
+
+import "google/protobuf/timestamp.proto";
+
+import "temporal/api/common/v1/message.proto";
+import "temporal/api/enums/v1/stream.proto";
+
+// Experimental. Might change in the future.
+// Identifies one Stream. The id is opaque to callers.
+message StreamRef {
+    string id = 1;
+    // Unset for a standalone Stream. For a Stream bound to a Workflow, business_id is the Workflow
+    // ID and run_id is the first execution run id of its run chain. For a Standalone Activity,
+    // business_id is the activity ID and run_id is its run id. A create and a close of a
+    // Workflow-owned Stream must carry its owner.
+    temporal.api.common.v1.Execution owner = 2;
+    // The name of the backend that holds the Stream, as the Worker and client config register
+    // it, so every process routes a ref it receives. Empty is the default backend. The Store
+    // never reads it.
+    string backend = 3;
+}
+
+// Experimental. Might change in the future.
+// The ordered attempt of a Producer. A higher epoch supersedes the Records of a lower epoch of the
+// same Producer. The case is the kind of the Producer. It decides how the Store orders the epochs,
+// and which earlier Records a higher epoch withdraws.
+message ProducerEpoch {
+    oneof kind {
+        // A Workflow run, which writes in its Workflow Task. A higher epoch withdraws the Records
+        // of the run's Workflow Tasks that never completed, also those of their Local Activities.
+        WorkflowTaskEpoch workflow = 1;
+        // A Local Activity. A higher epoch withdraws the Producer's lower epochs.
+        LocalActivityEpoch local_activity = 2;
+        // An Activity of a Workflow. A higher epoch withdraws the Producer's lower epochs.
+        ActivityEpoch activity = 3;
+        // A Standalone Activity. A higher epoch withdraws the Producer's lower epochs.
+        ActivityEpoch standalone_activity = 4;
+        // A client. A higher epoch only fences the lower ones, since a restarted client's earlier
+        // Records are real messages, not the Records of a failed attempt.
+        ClientEpoch client = 5;
+    }
+}
+
+// Experimental. Might change in the future.
+// One Workflow Task attempt of a run. It orders by started_event_id, then attempt, then
+// started_time.
+message WorkflowTaskEpoch {
+    int64 started_event_id = 1;
+    int32 attempt = 2;
+    // Tells a reused started event id apart, for example after a dropped speculative task.
+    google.protobuf.Timestamp started_time = 3;
+}
+
+// Experimental. Might change in the future.
+// The attempt of a Local Activity. It orders by task, then attempt, so a Local Activity of a
+// later Workflow Task supersedes one of an earlier task.
+message LocalActivityEpoch {
+    // The Workflow Task attempt that dispatched the Local Activity.
+    WorkflowTaskEpoch task = 1;
+    // The Local Activity's own attempt in that Workflow Task.
+    int32 attempt = 2;
+}
+
+// Experimental. Might change in the future.
+// The attempt of an Activity. It orders by attempt_scheduled_time, then attempt.
+message ActivityEpoch {
+    // When the server scheduled this attempt. An Activity reset starts the attempt count again,
+    // so only this time orders the attempts across a reset.
+    google.protobuf.Timestamp attempt_scheduled_time = 1;
+    int32 attempt = 2;
+}
+
+// Experimental. Might change in the future.
+// The attempt of a client Producer, which the client picks.
+message ClientEpoch {
+    int32 attempt = 1;
+}
+
+// Experimental. Might change in the future.
+// The code that wrote a batch. Its id stays the same across attempts, and its epoch orders them.
+message Producer {
+    // Stable across attempts, such as `wf:<run id>` for the Workflow Producer of a run.
+    string id = 1;
+    // Required. Its case gives the kind of the Producer.
+    ProducerEpoch epoch = 2;
+    // The run of the Producer, so nobody parses the id. Empty for a client.
+    string run_id = 3;
+}
+
+// Experimental. Might change in the future.
+// One Record of a batch to append. The Store adds the batch's Producer and an offset.
+message AppendRecord {
+    temporal.api.common.v1.Payload body = 1;
+}
+
+// Experimental. Might change in the future.
+// Which earlier Records a retried Producer withdraws. The Store keeps these facts for the first
+// batch of a higher epoch, or for a retry run's first write, and a read builds a Superseded item
+// from them. A reader drops the earlier Records that the facts match.
+message StreamSuperseded {
+    // The id of the Producer that withdraws.
+    string producer_id = 1;
+    // The epoch of the Producer that withdraws, whose case picks the rule. Empty when only
+    // withdraw_run applies.
+    ProducerEpoch epoch = 2;
+    // Workflow only. The run's last completed Workflow Task.
+    WorkflowTaskEpoch previous_completed_task = 3;
+    // Workflow only. The started event ids of the run's failed or timed out Workflow Tasks.
+    repeated int64 failed_tasks = 4;
+    // Workflow retry only. The previous run. Every Record whose Producer carries it is withdrawn.
+    string withdraw_run = 5;
+    // The run of the Producer, so nobody parses the id. Empty for a client.
+    string run_id = 6;
+}
+
+// Experimental. Might change in the future.
+// One Record that a Producer appended.
+message StreamData {
+    Producer producer = 1;
+    temporal.api.common.v1.Payload body = 2;
+}
+
+// Experimental. Might change in the future.
+// An offset is ordered and unique in one Stream, and it is sparse. Offsets have gaps, so they
+// don't count Records. Zero is never an offset. A read gives its Stream, so the read checks the
+// offset against that Stream.
+message StreamRecord {
+    int64 offset = 1;
+    // A read item is a Record or a Superseded item. A close writes no item.
+    oneof item {
+        StreamData data = 2;
+        // The Store keeps only the withdrawal facts of the first batch of a higher epoch. A read
+        // builds this item from them. It withdraws the earlier Records that the facts match, and a
+        // reader drops them.
+        StreamSuperseded superseded = 3;
+    }
+}
+
+// Experimental. Might change in the future.
+// The details of a failed Stream call, in the gRPC status details and in the details of the
+// application failure of a Nexus operation.
+message StreamErrorDetails {
+    temporal.api.enums.v1.StreamErrorReason reason = 1;
+}