diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b4a282..98eba6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,10 +39,10 @@ jobs: version: 1.4.2 - run: npm ci - run: npm run opa:generate - - name: Verify generated Rego is committed + - name: Verify generated OPA artifacts are committed run: | - git diff --exit-code -- opa/policies - test -z "$(git ls-files --others --exclude-standard opa/policies)" + git diff --exit-code -- opa + test -z "$(git ls-files --others --exclude-standard opa)" - run: npm run opa:check - run: npm run opa:test - run: npm run opa:server:test diff --git a/README.md b/README.md index 3d9d5a7..2e5443d 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,11 @@ npm exec --no -- prd validate --strict - `init` com agente, instruções, cinco templates de skills e workflow de CI para Copilot. - Migração aditiva do nosso protótipo anterior e CLI legado isolado. -O runtime `reference` usa a semântica nativa de `evaluateDecision`; `--runtime opa` gera Rego temporário e requer o executável `opa` disponível no PATH (ou em `PRD_OPA_BINARY`). OPA é um runtime candidato de conformance, não a autoridade semântica do modelo. +O runtime `reference` usa a semântica nativa de `evaluateDecision`; `--runtime +opa` recompila somente o subconjunto `dmn-table/v1` em Rego e requer o executável +`opa` no PATH (ou em `PRD_OPA_BINARY`). OPA é um candidato de conformidade, não +executa o produto e não substitui a autoridade semântica do domínio. Use +`--parity` para comparar também os contratos de entrada inválida, gap e overlap. A [matriz de capacidades](framework/capabilities.json) é o contrato de escopo. **Não há conformidade OMG completa, motor BPMN, FEEL, prova modal, execução de step definitions, servidor MCP ou autenticação de revisores nesta versão.** @@ -118,7 +122,7 @@ Cenários Gherkin usam `SCN-...` e não duplicam um arquivo YAML de identidade. | `test --require-tests --require-bound-scenarios` | Executa casos de decisões e exige vínculos de cenários | | `decision evaluate ID --input '{...}'` | Avalia uma tabela suportada | | `decision analyze ID` | Analisa domínios finitos | -| `conformance decision ID --runtime reference|opa` | Compara um runtime executável com a semântica de referência | +| `conformance decision ID --runtime reference\|opa [--parity] [--policy arquivo.rego]` | Compara casos e, opcionalmente, contratos de erro com a referência | | `cite ID --into notes/design.md` | Preserva snapshot e cita conteúdo | | `citations check`, `citations refresh arquivo.md` | Verifica ou atualiza citações intactas | | `change new SLUG --title MOTIVO` | Captura baseline | diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f31b9b8..ce8ab17 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,5 +1,7 @@ trigger: - main +variables: + OPA_VERSION: '1.4.2' pool: vmImage: ubuntu-latest steps: @@ -14,5 +16,22 @@ steps: displayName: Unit, integration and legacy tests - script: npm run intent:validate && npm run intent:test displayName: Validate product example + - script: | + curl --fail --location --silent --show-error \ + --output "$(Agent.TempDirectory)/opa" \ + "https://openpolicyagent.org/downloads/v$(OPA_VERSION)/opa_linux_amd64_static" + chmod 0755 "$(Agent.TempDirectory)/opa" + echo "##vso[task.prependpath]$(Agent.TempDirectory)" + displayName: Install OPA $(OPA_VERSION) + - script: | + npm run opa:generate + git diff --exit-code -- opa + git ls-files --others --exclude-standard opa > "$(Agent.TempDirectory)/opa-untracked" + test ! -s "$(Agent.TempDirectory)/opa-untracked" + npm run opa:check + npm run opa:test + npm run opa:server:test + npm run opa:conformance + displayName: OPA generation, server and parity tests - script: npm pack displayName: Package CLI diff --git a/docs/TESTING.md b/docs/TESTING.md index 8d18146..112df6e 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -12,7 +12,8 @@ | `npm test` | regressão automatizada do CLI, compilador e domínio | execução de uma aplicação externa | | `prd validate --strict` | schemas, referências e invariantes dos perfis | conformidade OMG completa | | `prd test` | resultados dos casos de decisões tipadas | execução dos passos Gherkin | -| `prd conformance decision ID --runtime opa` | equivalência dos casos entre a semântica de referência e o adapter OPA | equivalência para entradas fora dos casos declarados | +| `prd conformance decision ID --runtime opa` | equivalência dos casos declarados entre a referência e o adapter OPA | semântica fora do subconjunto `dmn-table/v1` | +| `prd conformance decision ID --runtime opa --parity` | paridade para chave extra/ausente, tipo, domínio, gap e overlap | conformidade DMN geral ou entradas arbitrárias | | `prd citations check` | estado dos blocos e snapshots conhecidos | autenticidade do autor | Ao adicionar um tipo ou alterar um contrato, atualize em conjunto o schema, o @@ -48,17 +49,28 @@ runtime candidato sem mover a autoridade semântica para esse runtime. ```sh prd conformance decision DEC-001 --runtime reference prd conformance decision DEC-001 --runtime opa +prd conformance decision DEC-001 --runtime opa --parity ``` O adapter OPA compila somente o subconjunto já suportado: tabela `UNIQUE`, condições de igualdade, wildcard por condição omitida e saída escalar. Ele não -expande o perfil DMN. O executável `opa` é opcional e não é baixado pelo pacote. +executa o produto nem expande o perfil DMN. Antes de chamar o binário, o adapter +aplica a mesma validação de nomes, tipos e domínios de entrada usada por +`evaluateDecision()`. O Rego também emite esse contrato e estados distintos para +`unique`, `gap`, `overlap` e `invalid_input`. + +Uma avaliação normal gera a policy uma vez e envia todos os casos em um único +`opa eval`, limitado por `PRD_OPA_TIMEOUT_MS` (5 segundos por padrão). O resultado +de conformidade registra a versão do binário. `--policy arquivo.rego` exige que o +artefato informado seja idêntico ao gerador para a especificação principal; +policies sintéticas das sondas de paridade continuam temporárias. ### Suíte OPA local e no GitHub Actions -A política versionada em `opa/policies/dec_001.rego` é gerada a partir de -`examples/transfer/product/decisions/DEC-001.yaml`. Para reproduzir a suíte do -CI localmente, instale o executável `opa` no `PATH` e execute: +A policy e o teste versionados em `opa/policies/` e `opa/tests/` são gerados a +partir de `examples/transfer/product/decisions/DEC-001.yaml`. O sufixo hash do +package impede que ids como `DEC-001` e `DEC_001` colidam. Para reproduzir a +suíte do CI localmente, instale OPA 1.4.2 no `PATH` e execute: ```sh npm ci @@ -70,9 +82,9 @@ npm run opa:conformance ``` `opa:check` valida sintaxe estrita e formatação. `opa:test` executa os testes -unitários Rego. `opa:server:test` inicia -temporariamente `opa run --server`, aguarda o health check e valida os quatro -casos pelo endpoint REST `/v1/data/prd/decision/dec_001/result`. -`opa:conformance` exercita o adapter do CLI contra a mesma decisão. O job -`OPA / Rego` também regenera a política e falha quando o arquivo versionado está -desatualizado em relação ao YAML. +Rego gerados, inclusive entradas inválidas. `opa:server:test` reserva uma porta +local livre, inicia temporariamente `opa run --server` e deriva casos, endpoint e +resultados esperados do mesmo YAML. `opa:conformance` usa a policy commitada para +os casos principais, ativa `--parity` e ainda cobre números `0`, `-1`, decimal, +inteiro acima de `2^53` e strings com aspas. GitHub Actions e Azure Pipelines +instalam OPA 1.4.2, regeneram `opa/` e falham se qualquer artefato divergir. diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index ec72f19..275f4f2 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -1,9 +1,12 @@ # Verificação da entrega -Executada em 2026-09-27, Linux, Node.js 24.19.0, npm 11.9.0. +Executada em 2026-10-04, Linux, Node.js 24.19.0, npm 11.9.0 e OPA 1.4.2. - TypeScript strict e fronteiras domain/application: passaram. -- `npm test`: 50 testes, 50 passaram, zero falhas (23 legado, 27 nativos). +- `npm test`: 64 testes, 64 passaram, zero falhas. +- OPA/Rego: formatação e check estrito passaram; 2 testes Rego passaram; 4 + casos passaram pelo servidor HTTP; paridade OPA passou em 11/11 sondas do + exemplo, 4/4 casos numéricos e 1/1 caso com aspas. - Exemplo integrado: 11 artefatos, 3 cenários e 4 casos de decisão; validação estrita e casos passaram. - Demo: init, validação, citação, proposta, aprovação, aplicação, stale e refresh/current passaram. - Pacote npm gerado e instalado em diretório independente. Binário instalado criou projeto em inglês com kit Copilot, validou estritamente e executou os 4 casos com todos os cenários vinculados. @@ -11,6 +14,8 @@ Executada em 2026-09-27, Linux, Node.js 24.19.0, npm 11.9.0. - Links locais da documentação: resolvidos. - Duas citações nativas no exemplo: current. -Não executados nesta sessão: sessão real do Copilot, CI remoto, Windows/macOS ou Node 22. A matriz de CI declara Node 22/24, sem afirmar resultado de jobs remotos. Não há publicação no npm nem configuração de repositório remoto. +Não executados nesta sessão: sessão real do Copilot, CI remoto desta branch, +Azure Pipelines, Windows/macOS ou Node 22. A matriz de CI declara Node 22/24, +sem afirmar resultado de jobs remotos. Não há publicação no npm. A suíte não é prova de conformidade OMG ou correção de processos em execução. Consulte framework/capabilities.json e os perfis para o escopo exato. diff --git a/framework/capabilities.json b/framework/capabilities.json index 19e57fa..94af066 100644 --- a/framework/capabilities.json +++ b/framework/capabilities.json @@ -52,7 +52,8 @@ "finite domain exhaustiveness up to 4096 combinations", "scenario-linked cases", "reference runtime conformance contract", - "optional OPA/Rego adapter for declared decision cases" + "optional OPA/Rego adapter for declared decision cases", + "OPA input, gap and overlap error parity probes" ], "unsupported": [ "FEEL", diff --git a/opa/policies/dec_001.rego b/opa/policies/dec_001.rego deleted file mode 100644 index 39ec53d..0000000 --- a/opa/policies/dec_001.rego +++ /dev/null @@ -1,20 +0,0 @@ -package prd.decision.dec_001 - -matches contains {"value": "CONTA_INATIVA", "ruleId": "DROW-001"} if { - input.contaAtiva == false -} - -matches contains {"value": "SALDO_INSUFICIENTE", "ruleId": "DROW-002"} if { - input.contaAtiva == true - input.saldoSuficiente == false -} - -matches contains {"value": "ELEGIVEL", "ruleId": "DROW-003"} if { - input.contaAtiva == true - input.saldoSuficiente == true -} - -result := item if { - count(matches) == 1 - item := matches[_] -} diff --git a/opa/policies/dec_001_e3d5dad9367a.rego b/opa/policies/dec_001_e3d5dad9367a.rego new file mode 100644 index 0000000..bba5c4d --- /dev/null +++ b/opa/policies/dec_001_e3d5dad9367a.rego @@ -0,0 +1,81 @@ +package prd.decision.dec_001_e3d5dad9367a + +decision_id := "DEC-001" + +input_names := ["contaAtiva", "saldoSuficiente"] + +valid_input if { + is_object(input) + count(input) == count(input_names) + every name in input_names { + object.get(input, name, {"missing": true}) != {"missing": true} + } + is_boolean(input.contaAtiva) + is_boolean(input.saldoSuficiente) +} + +output_valid(value) if { + is_string(value) + value in ["ELEGIVEL", "CONTA_INATIVA", "SALDO_INSUFICIENTE"] +} + +# Keeps the matches set defined for an empty or fully filtered rule table. +matches contains {"value": false, "ruleId": ""} if { + false +} + +matches contains {"value": "CONTA_INATIVA", "ruleId": "DROW-001"} if { + valid_input + input.contaAtiva == false +} + +matches contains {"value": "SALDO_INSUFICIENTE", "ruleId": "DROW-002"} if { + valid_input + input.contaAtiva == true + input.saldoSuficiente == false +} + +matches contains {"value": "ELEGIVEL", "ruleId": "DROW-003"} if { + valid_input + input.contaAtiva == true + input.saldoSuficiente == true +} + +evaluation := {"status": "invalid_input"} if { + not valid_input +} + +evaluation := {"status": "gap", "ruleIds": []} if { + valid_input + count(matches) == 0 +} + +evaluation := {"status": "overlap", "ruleIds": rule_ids} if { + valid_input + count(matches) > 1 + rule_ids := sort([rule.ruleId | some rule in matches]) +} + +evaluation := {"status": "invalid_output", "ruleId": item.ruleId} if { + valid_input + count(matches) == 1 + item := matches[_] + not output_valid(item.value) +} + +evaluation := {"status": "unique", "value": item.value, "ruleId": item.ruleId} if { + valid_input + count(matches) == 1 + item := matches[_] + output_valid(item.value) +} + +result := item if { + evaluation.status == "unique" + item := {"value": evaluation.value, "ruleId": evaluation.ruleId} +} + +batch := [outcome | + some item in input + outcome := evaluation with input as item +] diff --git a/opa/tests/dec_001_e3d5dad9367a_test.rego b/opa/tests/dec_001_e3d5dad9367a_test.rego new file mode 100644 index 0000000..60f94e9 --- /dev/null +++ b/opa/tests/dec_001_e3d5dad9367a_test.rego @@ -0,0 +1,32 @@ +package prd.decision.dec_001_e3d5dad9367a_test + +import data.prd.decision.dec_001_e3d5dad9367a.evaluation + +decision_cases := [ + {"id": "CASE-001", "input": {"contaAtiva": true, "saldoSuficiente": true}, "expected": {"value": "ELEGIVEL", "ruleId": "DROW-003"}}, + {"id": "CASE-002", "input": {"contaAtiva": false, "saldoSuficiente": true}, "expected": {"value": "CONTA_INATIVA", "ruleId": "DROW-001"}}, + {"id": "CASE-003", "input": {"contaAtiva": true, "saldoSuficiente": false}, "expected": {"value": "SALDO_INSUFICIENTE", "ruleId": "DROW-002"}}, + {"id": "CASE-004", "input": {"contaAtiva": false, "saldoSuficiente": false}, "expected": {"value": "CONTA_INATIVA", "ruleId": "DROW-001"}}, +] + +invalid_inputs := [ + {"contaAtiva": true, "saldoSuficiente": true, "__unexpected": true}, + {"saldoSuficiente": true}, + {"contaAtiva": "__wrong_type__", "saldoSuficiente": true}, +] + +test_declared_decision_cases if { + every case in decision_cases { + actual := evaluation with input as case.input + actual.status == "unique" + actual.value == case.expected.value + actual.ruleId == case.expected.ruleId + } +} + +test_input_contract if { + every invalid_input in invalid_inputs { + actual := evaluation with input as invalid_input + actual.status == "invalid_input" + } +} diff --git a/opa/tests/dec_001_test.rego b/opa/tests/dec_001_test.rego deleted file mode 100644 index 1dbb5d6..0000000 --- a/opa/tests/dec_001_test.rego +++ /dev/null @@ -1,37 +0,0 @@ -package prd.decision.dec_001_test - -import data.prd.decision.dec_001.result - -cases := [ - { - "id": "CASE-001", - "input": {"contaAtiva": true, "saldoSuficiente": true}, - "expected": {"value": "ELEGIVEL", "ruleId": "DROW-003"}, - }, - { - "id": "CASE-002", - "input": {"contaAtiva": false, "saldoSuficiente": true}, - "expected": {"value": "CONTA_INATIVA", "ruleId": "DROW-001"}, - }, - { - "id": "CASE-003", - "input": {"contaAtiva": true, "saldoSuficiente": false}, - "expected": {"value": "SALDO_INSUFICIENTE", "ruleId": "DROW-002"}, - }, - { - "id": "CASE-004", - "input": {"contaAtiva": false, "saldoSuficiente": false}, - "expected": {"value": "CONTA_INATIVA", "ruleId": "DROW-001"}, - }, -] - -test_declared_decision_cases if { - every case in cases { - actual := result with input as case.input - actual == case.expected - } -} - -test_incomplete_input_is_undefined if { - not result with input as {"contaAtiva": true} -} diff --git a/package.json b/package.json index 3906557..633b7f9 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "opa:check": "opa fmt --list --fail opa/policies opa/tests && opa check --strict opa/policies opa/tests", "opa:test": "opa test opa/policies opa/tests --verbose", "opa:server:test": "node scripts/test-opa-server.mjs", - "opa:conformance": "node dist/interfaces/cli/main.js -C examples/transfer conformance decision DEC-001 --runtime opa" + "opa:conformance": "node scripts/test-opa-conformance.mjs" }, "dependencies": { "commander": "14.0.2", "yaml": "2.9.1", "ajv": "8.17.1", diff --git a/scripts/generate-opa-policies.mjs b/scripts/generate-opa-policies.mjs index d9ec00c..35254d1 100644 --- a/scripts/generate-opa-policies.mjs +++ b/scripts/generate-opa-policies.mjs @@ -2,14 +2,19 @@ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { parse } from 'yaml'; -import { generateDecisionRego, opaPackageName } from '../dist/infrastructure/opa/rego-generator.js'; +import { + generateDecisionRego, + generateDecisionTestRego, + opaPackageName, +} from '../dist/infrastructure/opa/rego-generator.js'; const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const sourcePath = resolve( repositoryRoot, process.argv[2] ?? 'examples/transfer/product/decisions/DEC-001.yaml', ); -const targetDirectory = resolve(repositoryRoot, process.argv[3] ?? 'opa/policies'); +const policyDirectory = resolve(repositoryRoot, process.argv[3] ?? 'opa/policies'); +const testDirectory = resolve(repositoryRoot, process.argv[4] ?? 'opa/tests'); const artifact = parse(readFileSync(sourcePath, 'utf8')); if (artifact?.kind !== 'Decision' || typeof artifact.metadata?.id !== 'string' || !artifact.spec) { @@ -18,8 +23,11 @@ if (artifact?.kind !== 'Decision' || typeof artifact.metadata?.id !== 'string' | const packageName = opaPackageName(artifact.metadata.id); const fileName = `${packageName.split('.').at(-1)}.rego`; -const targetPath = resolve(targetDirectory, fileName); +const policyPath = resolve(policyDirectory, fileName); +const testPath = resolve(testDirectory, fileName.replace(/\.rego$/, '_test.rego')); -mkdirSync(targetDirectory, { recursive: true }); -writeFileSync(targetPath, generateDecisionRego(artifact.metadata.id, artifact.spec), 'utf8'); -process.stdout.write(`Generated ${targetPath}\n`); +mkdirSync(policyDirectory, { recursive: true }); +mkdirSync(testDirectory, { recursive: true }); +writeFileSync(policyPath, generateDecisionRego(artifact.metadata.id, artifact.spec), 'utf8'); +writeFileSync(testPath, generateDecisionTestRego(artifact.metadata.id, artifact.spec), 'utf8'); +process.stdout.write(`Generated ${policyPath}\nGenerated ${testPath}\n`); diff --git a/scripts/test-opa-conformance.mjs b/scripts/test-opa-conformance.mjs new file mode 100644 index 0000000..85080d4 --- /dev/null +++ b/scripts/test-opa-conformance.mjs @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { parse } from 'yaml'; +import { conformDecision } from '../dist/application/decision-conformance.js'; +import { OpaDecisionRuntime } from '../dist/infrastructure/opa/opa-decision-runtime.js'; +import { opaPackageName } from '../dist/infrastructure/opa/rego-generator.js'; + +const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const artifactPath = resolve( + repositoryRoot, + 'examples/transfer/product/decisions/DEC-001.yaml', +); +const artifact = parse(readFileSync(artifactPath, 'utf8')); +const packageName = opaPackageName(artifact.metadata.id); +const policyPath = resolve( + repositoryRoot, + 'opa/policies', + `${packageName.split('.').at(-1)}.rego`, +); +const result = spawnSync( + process.execPath, + [ + resolve(repositoryRoot, 'dist/interfaces/cli/main.js'), + '-C', + resolve(repositoryRoot, 'examples/transfer'), + 'conformance', + 'decision', + artifact.metadata.id, + '--runtime', + 'opa', + '--parity', + '--policy', + policyPath, + ], + { stdio: 'inherit', windowsHide: true }, +); + +if (result.error && result.status === null) throw result.error; +if (result.status !== 0) process.exit(result.status ?? 2); + +const numericValues = [0, -1, 1.25, 9007199254740994]; +const numericSpec = { + profile: 'dmn-table/v1', + hitPolicy: 'UNIQUE', + inputs: [{ name: 'amount', type: 'number', values: numericValues }], + output: { name: 'result', type: 'number', values: numericValues }, + rules: numericValues.map((value, index) => ({ + id: `number-${index}`, + when: { amount: value }, + then: value, + })), + cases: numericValues.map((value, index) => ({ + id: `N-${index}`, + input: { amount: value }, + expected: value, + })), +}; +const quotedSpec = { + profile: 'dmn-table/v1', + hitPolicy: 'UNIQUE', + inputs: [{ name: 'message', type: 'string' }], + output: { name: 'result', type: 'string' }, + rules: [{ id: 'quoted', when: { message: 'say "hello"' }, then: 'answer "ok"' }], + cases: [{ id: 'QUOTED-1', input: { message: 'say "hello"' }, expected: 'answer "ok"' }], +}; + +for (const [decisionId, spec] of [ + ['DEC-NUMBERS', numericSpec], + ['DEC-QUOTED', quotedSpec], +]) { + const conformance = await conformDecision(decisionId, spec, new OpaDecisionRuntime()); + assert.equal(conformance.passed, true, `${decisionId}: OPA diverged from reference semantics`); + process.stdout.write( + `${decisionId} on OPA ${conformance.runtimeVersion}: ${conformance.matched}/${conformance.cases.length} cases conform\n`, + ); +} diff --git a/scripts/test-opa-server.mjs b/scripts/test-opa-server.mjs index 51a1504..b0c63b6 100644 --- a/scripts/test-opa-server.mjs +++ b/scripts/test-opa-server.mjs @@ -1,13 +1,43 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { createServer } from 'node:net'; +import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { setTimeout as delay } from 'node:timers/promises'; +import { parse } from 'yaml'; +import { evaluateDecision } from '../dist/domain/decisions.js'; +import { opaPackageName } from '../dist/infrastructure/opa/rego-generator.js'; +const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const binary = process.env.PRD_OPA_BINARY || 'opa'; const host = process.env.OPA_TEST_HOST || '127.0.0.1'; -const port = Number(process.env.OPA_TEST_PORT || 8181); +const artifactPath = resolve( + repositoryRoot, + process.env.OPA_TEST_DECISION || 'examples/transfer/product/decisions/DEC-001.yaml', +); +const artifact = parse(readFileSync(artifactPath, 'utf8')); +if (artifact?.kind !== 'Decision' || typeof artifact.metadata?.id !== 'string' || !artifact.spec) { + throw new Error(`${artifactPath} is not a Decision artifact.`); +} + +async function reservePort() { + const listener = createServer(); + await new Promise((resolveListen, reject) => { + listener.once('error', reject); + listener.listen(0, host, resolveListen); + }); + const address = listener.address(); + if (!address || typeof address === 'string') throw new Error('Could not reserve an OPA test port.'); + await new Promise((resolveClose, reject) => listener.close(error => error ? reject(error) : resolveClose())); + return address.port; +} + +const port = Number(process.env.OPA_TEST_PORT || await reservePort()); const baseUrl = `http://${host}:${port}`; -const policyPath = fileURLToPath(new URL('../opa/policies', import.meta.url)); +const packageName = opaPackageName(artifact.metadata.id); +const policyPath = resolve(repositoryRoot, 'opa/policies', `${packageName.split('.').at(-1)}.rego`); +const decisionUrl = `${baseUrl}/v1/data/${packageName.replaceAll('.', '/')}/result`; const server = spawn(binary, ['run', '--server', `--addr=${host}:${port}`, policyPath], { stdio: ['ignore', 'pipe', 'pipe'], }); @@ -20,7 +50,7 @@ async function waitUntilReady() { for (let attempt = 0; attempt < 50; attempt += 1) { if (server.exitCode !== null) throw new Error(`OPA server stopped unexpectedly.\n${output}`); try { - const response = await fetch(`${baseUrl}/health?bundles`); + const response = await fetch(`${baseUrl}/health`); if (response.ok) return; } catch { // The listener may not be ready yet. @@ -30,46 +60,31 @@ async function waitUntilReady() { throw new Error(`OPA server did not become ready at ${baseUrl}.\n${output}`); } -const cases = [ - { - id: 'CASE-001', - input: { contaAtiva: true, saldoSuficiente: true }, - expected: { value: 'ELEGIVEL', ruleId: 'DROW-003' }, - }, - { - id: 'CASE-002', - input: { contaAtiva: false, saldoSuficiente: true }, - expected: { value: 'CONTA_INATIVA', ruleId: 'DROW-001' }, - }, - { - id: 'CASE-003', - input: { contaAtiva: true, saldoSuficiente: false }, - expected: { value: 'SALDO_INSUFICIENTE', ruleId: 'DROW-002' }, - }, - { - id: 'CASE-004', - input: { contaAtiva: false, saldoSuficiente: false }, - expected: { value: 'CONTA_INATIVA', ruleId: 'DROW-001' }, - }, -]; - try { - await waitUntilReady(); - for (const testCase of cases) { - const response = await fetch(`${baseUrl}/v1/data/prd/decision/dec_001/result`, { + await Promise.race([ + waitUntilReady(), + new Promise((_, reject) => server.once('error', reject)), + ]); + for (const testCase of artifact.spec.cases) { + const expected = evaluateDecision(artifact.spec, testCase.input); + const response = await fetch(decisionUrl, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ input: testCase.input }), }); assert.equal(response.status, 200, `${testCase.id}: unexpected HTTP status`); const document = await response.json(); - assert.deepEqual(document.result, testCase.expected, `${testCase.id}: divergent decision`); + assert.deepEqual(document.result, expected, `${testCase.id}: divergent decision`); } - process.stdout.write(`OPA server: ${cases.length} decision cases passed.\n`); + process.stdout.write( + `OPA server ${packageName}: ${artifact.spec.cases.length} generated decision cases passed.\n`, + ); } finally { - server.kill('SIGTERM'); - await Promise.race([ - new Promise(resolveExit => server.once('exit', resolveExit)), - delay(2_000, undefined, { ref: false }).then(() => server.kill('SIGKILL')), - ]); + if (server.pid && server.exitCode === null) { + server.kill('SIGTERM'); + await Promise.race([ + new Promise(resolveExit => server.once('exit', resolveExit)), + delay(2_000, undefined, { ref: false }).then(() => server.kill('SIGKILL')), + ]); + } } diff --git a/src/application/decision-conformance.ts b/src/application/decision-conformance.ts index b765552..6b8a7bd 100644 --- a/src/application/decision-conformance.ts +++ b/src/application/decision-conformance.ts @@ -1,14 +1,24 @@ import { evaluateDecision } from '../domain/decisions.js'; import type { DecisionSpec, Scalar } from '../domain/decisions.js'; -import type { DecisionRuntime, DecisionRuntimeResult } from './ports/decision-runtime.js'; +import { IntentError, sortedJson } from '../domain/model.js'; +import type { + DecisionRuntime, + DecisionRuntimeOutcome, + DecisionRuntimeRequest, + DecisionRuntimeResult, +} from './ports/decision-runtime.js'; export interface DecisionConformanceCase { id: string; + kind: 'declared' | 'parity'; scenario?: string; - input: Record; - expected: Scalar; + input: Record; + expected?: Scalar; + expectedError?: string; reference?: DecisionRuntimeResult; runtime?: DecisionRuntimeResult; + referenceError?: string; + runtimeError?: string; passed: boolean; error?: string; } @@ -16,57 +26,264 @@ export interface DecisionConformanceCase { export interface DecisionConformanceResult { decision: string; runtime: string; + runtimeVersion?: string; + parity: boolean; passed: boolean; cases: DecisionConformanceCase[]; matched: number; divergent: number; } +interface ConformanceProbe { + id: string; + kind: 'declared' | 'parity'; + scenario?: string; + spec: DecisionSpec; + input: Record; + expected?: Scalar; + expectedError?: string; +} + +function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +function errorCode(error: Error): string { + return error instanceof IntentError ? error.code : error.name; +} + +function scalarEqual(left: Scalar, right: Scalar): boolean { + return sortedJson(left) === sortedJson(right); +} + +function settleReference(probe: ConformanceProbe): DecisionRuntimeOutcome { + try { + return { ok: true, result: evaluateDecision(probe.spec, probe.input) }; + } catch (error) { + return { ok: false, error: asError(error) }; + } +} + +async function settleRuntime( + runtime: DecisionRuntime, + decisionId: string, + probes: ConformanceProbe[], +): Promise { + const outcomes: Array = new Array(probes.length); + const groups = new Map(); + for (const [index, probe] of probes.entries()) { + const key = sortedJson(probe.spec); + const group = groups.get(key) ?? []; + group.push({ + index, + request: { decisionId, spec: probe.spec, input: probe.input }, + }); + groups.set(key, group); + } + + for (const group of groups.values()) { + if (runtime.evaluateBatch) { + const batch = await runtime.evaluateBatch(group.map(item => item.request)); + if (batch.length !== group.length) { + throw new IntentError('RUNTIME_INVALID_OUTPUT', 'Runtime batch result count does not match.'); + } + for (const [index, outcome] of batch.entries()) outcomes[group[index]!.index] = outcome; + } else { + const settled = await Promise.all(group.map(async item => { + try { + return { ok: true, result: await runtime.evaluate(item.request) } as const; + } catch (error) { + return { ok: false, error: asError(error) } as const; + } + })); + for (const [index, outcome] of settled.entries()) outcomes[group[index]!.index] = outcome; + } + } + + return outcomes.map(outcome => outcome ?? ({ + ok: false, + error: new IntentError('RUNTIME_INVALID_OUTPUT', 'Runtime returned no outcome.'), + })); +} + +function alternativeValue( + input: { type: 'boolean' | 'number' | 'string' }, + current: Scalar, +): Scalar { + if (input.type === 'boolean') return current !== true; + if (input.type === 'number') return current === 0 ? 1 : 0; + const candidate = '__PRD_PARITY_OUTSIDE_DOMAIN__'; + return current === candidate ? `${candidate}_2` : candidate; +} + +function invalidOutputValue(spec: DecisionSpec): Scalar { + if (!spec.output.values) { + return spec.output.type === 'string' ? false : '__wrong_output_type__'; + } + const candidates: Scalar[] = spec.output.type === 'boolean' + ? [false, true] + : spec.output.type === 'number' + ? [0, -1, 1, 1.25, 9007199254740994] + : ['__PRD_PARITY_OUTSIDE_DOMAIN__', '__PRD_PARITY_OUTSIDE_DOMAIN_2__']; + return candidates.find(candidate => !spec.output.values!.includes(candidate)) ?? + (spec.output.type === 'string' ? false : '__wrong_output_type__'); +} + +function parityProbes(spec: DecisionSpec): ConformanceProbe[] { + const seed = spec.cases[0]?.input; + const firstInput = spec.inputs[0]; + if (!seed || !firstInput) return []; + + const missing = { ...seed } as Record; + delete missing[firstInput.name]; + const wrongType = { + ...seed, + [firstInput.name]: firstInput.type === 'string' ? false : '__wrong_type__', + }; + const matchingRule = spec.rules.find(rule => + Object.entries(rule.when).every(([key, value]) => seed[key] === value)); + const overlapSpec: DecisionSpec = { + ...spec, + rules: matchingRule + ? [...spec.rules, { ...matchingRule, id: '__PRD_PARITY_OVERLAP__' }] + : [...spec.rules, { id: '__PRD_PARITY_OVERLAP__', when: {}, then: spec.rules[0]!.then }], + }; + const domainSpec: DecisionSpec = { + ...spec, + inputs: spec.inputs.map((input, index) => index === 0 + ? { ...input, values: [seed[input.name] as Scalar] } + : input), + }; + const invalidOutput = invalidOutputValue(spec); + const invalidOutputSpec: DecisionSpec = { + ...spec, + rules: spec.rules.map(rule => rule.id === matchingRule?.id + ? { ...rule, then: invalidOutput } + : rule), + }; + + return [ + { + id: 'PARITY-EXTRA-INPUT', + kind: 'parity', + spec, + input: { ...seed, __unexpected: true }, + expectedError: 'INVALID_INPUT', + }, + { + id: 'PARITY-MISSING-INPUT', + kind: 'parity', + spec, + input: missing, + expectedError: 'INVALID_INPUT', + }, + { + id: 'PARITY-WRONG-TYPE', + kind: 'parity', + spec, + input: wrongType, + expectedError: 'INVALID_INPUT', + }, + { + id: 'PARITY-OUTSIDE-DOMAIN', + kind: 'parity', + spec: domainSpec, + input: { + ...seed, + [firstInput.name]: alternativeValue(firstInput, seed[firstInput.name] as Scalar), + }, + expectedError: 'INVALID_INPUT', + }, + { + id: 'PARITY-GAP', + kind: 'parity', + spec: { ...spec, rules: [] }, + input: seed, + expectedError: 'DECISION_GAP', + }, + { + id: 'PARITY-OVERLAP', + kind: 'parity', + spec: overlapSpec, + input: seed, + expectedError: 'DECISION_OVERLAP', + }, + { + id: 'PARITY-INVALID-OUTPUT', + kind: 'parity', + spec: invalidOutputSpec, + input: seed, + expectedError: 'INVALID_DECISION_OUTPUT', + }, + ]; +} + export async function conformDecision( decisionId: string, spec: DecisionSpec, runtime: DecisionRuntime, + options: { parity?: boolean } = {}, ): Promise { - const cases: DecisionConformanceCase[] = []; - - for (const testCase of spec.cases) { - try { - const reference = evaluateDecision(spec, testCase.input); - const candidate = await runtime.evaluate({ - decisionId, - spec, - input: testCase.input, - }); - const passed = - Object.is(reference.value, testCase.expected) && - Object.is(candidate.value, reference.value) && - candidate.ruleId === reference.ruleId; - - cases.push({ - id: testCase.id, - scenario: testCase.scenario, - input: testCase.input, - expected: testCase.expected, - reference, - runtime: candidate, - passed, - }); - } catch (error) { - cases.push({ - id: testCase.id, - scenario: testCase.scenario, - input: testCase.input, - expected: testCase.expected, - passed: false, - error: (error as Error).message, - }); + const probes: ConformanceProbe[] = [ + ...spec.cases.map(testCase => ({ + id: testCase.id, + kind: 'declared' as const, + scenario: testCase.scenario, + spec, + input: testCase.input, + expected: testCase.expected, + })), + ...(options.parity ? parityProbes(spec) : []), + ]; + const runtimeOutcomes = await settleRuntime(runtime, decisionId, probes); + const cases = probes.map((probe, index): DecisionConformanceCase => { + const reference = settleReference(probe); + const candidate = runtimeOutcomes[index]!; + if (probe.expectedError) { + const referenceError = reference.ok ? undefined : errorCode(reference.error); + const runtimeError = candidate.ok ? undefined : errorCode(candidate.error); + return { + id: probe.id, + kind: probe.kind, + scenario: probe.scenario, + input: probe.input, + expectedError: probe.expectedError, + reference: reference.ok ? reference.result : undefined, + runtime: candidate.ok ? candidate.result : undefined, + referenceError, + runtimeError, + passed: referenceError === probe.expectedError && runtimeError === referenceError, + }; } - } - const matched = cases.filter(c => c.passed).length; + const passed = reference.ok && candidate.ok && probe.expected !== undefined && + scalarEqual(reference.result.value, probe.expected) && + scalarEqual(candidate.result.value, reference.result.value) && + candidate.result.ruleId === reference.result.ruleId; + return { + id: probe.id, + kind: probe.kind, + scenario: probe.scenario, + input: probe.input, + expected: probe.expected, + reference: reference.ok ? reference.result : undefined, + runtime: candidate.ok ? candidate.result : undefined, + referenceError: reference.ok ? undefined : errorCode(reference.error), + runtimeError: candidate.ok ? undefined : errorCode(candidate.error), + passed, + error: reference.ok && candidate.ok + ? undefined + : [reference.ok ? undefined : reference.error.message, candidate.ok ? undefined : candidate.error.message] + .filter(Boolean).join(' | '), + }; + }); + + const matched = cases.filter(testCase => testCase.passed).length; return { decision: decisionId, runtime: runtime.name, + runtimeVersion: runtime.version?.(), + parity: Boolean(options.parity), passed: cases.length > 0 && matched === cases.length, cases, matched, diff --git a/src/application/ports/decision-runtime.ts b/src/application/ports/decision-runtime.ts index 0e579ed..841d8fa 100644 --- a/src/application/ports/decision-runtime.ts +++ b/src/application/ports/decision-runtime.ts @@ -3,7 +3,7 @@ import type { DecisionSpec, Scalar } from '../../domain/decisions.js'; export interface DecisionRuntimeRequest { decisionId: string; spec: DecisionSpec; - input: Record; + input: Record; } export interface DecisionRuntimeResult { @@ -11,7 +11,13 @@ export interface DecisionRuntimeResult { ruleId: string; } +export type DecisionRuntimeOutcome = + | { ok: true; result: DecisionRuntimeResult } + | { ok: false; error: Error }; + export interface DecisionRuntime { readonly name: string; evaluate(request: DecisionRuntimeRequest): Promise; + evaluateBatch?(requests: DecisionRuntimeRequest[]): Promise; + version?(): string | undefined; } diff --git a/src/domain/decisions.ts b/src/domain/decisions.ts index 9f47dc9..1b0065f 100644 --- a/src/domain/decisions.ts +++ b/src/domain/decisions.ts @@ -7,11 +7,37 @@ export interface DecisionSpec { rules: { id: string; when: Record; then: Scalar }[]; cases: { id: string; scenario?: string; input: Record; expected: Scalar }[]; } + +export function validateDecisionInput( + spec: DecisionSpec, + input: unknown, +): asserts input is Record { + if (!input || typeof input !== 'object' || Array.isArray(input) || + Object.keys(input).length !== spec.inputs.length || + spec.inputs.some(i => typeof (input as Record)[i.name] !== i.type || + (i.type === 'number' && !Number.isFinite((input as Record)[i.name])) || + (i.values && !i.values.includes((input as Record)[i.name] as Scalar)))) { + throw new IntentError( + 'INVALID_INPUT', + 'Decision inputs must exactly match declared names, types, and domains.', + ); + } +} + export function evaluateDecision(spec: DecisionSpec, input: Record) { - if (!input || typeof input !== 'object' || Array.isArray(input) || Object.keys(input).length !== spec.inputs.length || spec.inputs.some(i => typeof input[i.name] !== i.type || (i.type === 'number' && !Number.isFinite(input[i.name])) || (i.values && !i.values.includes(input[i.name] as Scalar)))) throw new IntentError('INVALID_INPUT', 'Decision inputs must exactly match declared names, types, and domains.'); + validateDecisionInput(spec, input); const matches = spec.rules.filter(r => Object.entries(r.when).every(([key, value]) => input[key] === value)); if (matches.length !== 1) throw new IntentError(matches.length ? 'DECISION_OVERLAP' : 'DECISION_GAP', `UNIQUE decision matched ${matches.length} rows.`); - return { value: matches[0]!.then, ruleId: matches[0]!.id }; + const value = matches[0]!.then; + if (typeof value !== spec.output.type || + (spec.output.type === 'number' && !Number.isFinite(value)) || + (spec.output.values && !spec.output.values.includes(value))) { + throw new IntentError( + 'INVALID_DECISION_OUTPUT', + 'Decision output must match its declared type and domain.', + ); + } + return { value, ruleId: matches[0]!.id }; } export function analyzeDecision(spec: DecisionSpec) { const domains = spec.inputs.map(i => i.values ?? (i.type === 'boolean' ? [false, true] : undefined)); diff --git a/src/infrastructure/opa/opa-decision-runtime.ts b/src/infrastructure/opa/opa-decision-runtime.ts index c87b6c0..56550ac 100644 --- a/src/infrastructure/opa/opa-decision-runtime.ts +++ b/src/infrastructure/opa/opa-decision-runtime.ts @@ -1,97 +1,228 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { spawnSync } from 'node:child_process'; +import { execFile, spawnSync } from 'node:child_process'; +import { promisify } from 'node:util'; import { IntentError, isRecord } from '../../domain/model.js'; -import type { Scalar } from '../../domain/decisions.js'; +import { validateDecisionInput } from '../../domain/decisions.js'; +import type { DecisionSpec, Scalar } from '../../domain/decisions.js'; import type { DecisionRuntime, + DecisionRuntimeOutcome, DecisionRuntimeRequest, DecisionRuntimeResult, } from '../../application/ports/decision-runtime.js'; import { generateDecisionRego, opaPackageName } from './rego-generator.js'; +const DEFAULT_TIMEOUT_MS = 5_000; +const execFileAsync = promisify(execFile); + function isScalar(value: unknown): value is Scalar { return typeof value === 'string' || typeof value === 'boolean' || (typeof value === 'number' && Number.isFinite(value)); } +function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +function evaluationOutcome(value: unknown): DecisionRuntimeOutcome { + if (!isRecord(value) || typeof value.status !== 'string') { + return { ok: false, error: new IntentError('OPA_INVALID_OUTPUT', 'OPA evaluation status is missing.') }; + } + if (value.status === 'unique') { + if (!isScalar(value.value) || typeof value.ruleId !== 'string') { + return { + ok: false, + error: new IntentError('OPA_INVALID_OUTPUT', 'OPA decision result has invalid value or ruleId.'), + }; + } + return { ok: true, result: { value: value.value, ruleId: value.ruleId } }; + } + if (value.status === 'invalid_input') { + return { + ok: false, + error: new IntentError( + 'INVALID_INPUT', + 'Decision inputs must exactly match declared names, types, and domains.', + ), + }; + } + if (value.status === 'invalid_output') { + return { + ok: false, + error: new IntentError( + 'INVALID_DECISION_OUTPUT', + 'Decision output must match its declared type and domain.', + { ruleId: typeof value.ruleId === 'string' ? value.ruleId : undefined }, + ), + }; + } + const ruleIds = Array.isArray(value.ruleIds) + ? value.ruleIds.filter((item): item is string => typeof item === 'string') + : []; + if (value.status === 'gap') { + return { + ok: false, + error: new IntentError('DECISION_GAP', 'UNIQUE decision matched 0 rows.', { ruleIds }), + }; + } + if (value.status === 'overlap') { + return { + ok: false, + error: new IntentError( + 'DECISION_OVERLAP', + `UNIQUE decision matched ${ruleIds.length} rows.`, + { ruleIds }, + ), + }; + } + return { + ok: false, + error: new IntentError('OPA_INVALID_OUTPUT', `Unknown OPA evaluation status: ${value.status}.`), + }; +} + export class OpaDecisionRuntime implements DecisionRuntime { readonly name = 'opa'; - constructor(readonly binary = process.env.PRD_OPA_BINARY || 'opa') {} + constructor( + readonly binary = process.env.PRD_OPA_BINARY || 'opa', + readonly policyPath?: string, + readonly timeoutMs = Number(process.env.PRD_OPA_TIMEOUT_MS || DEFAULT_TIMEOUT_MS), + readonly policyBaseline?: { decisionId: string; spec: DecisionSpec }, + ) {} + + version(): string | undefined { + const result = spawnSync(this.binary, ['version'], { + encoding: 'utf8', + timeout: this.timeoutMs, + windowsHide: true, + }); + if (result.status !== 0) return undefined; + return /^Version:\s*(.+)$/m.exec(result.stdout)?.[1]?.trim(); + } async evaluate(request: DecisionRuntimeRequest): Promise { - const dir = mkdtempSync(join(tmpdir(), 'prd-opa-')); - const policy = join(dir, 'decision.rego'); + const [outcome] = await this.evaluateBatch([request]); + if (!outcome) throw new IntentError('OPA_INVALID_OUTPUT', 'OPA returned no decision outcome.'); + if (!outcome.ok) throw outcome.error; + return outcome.result; + } - try { - writeFileSync(policy, generateDecisionRego(request.decisionId, request.spec), 'utf8'); - const query = `data.${opaPackageName(request.decisionId)}.result`; - const result = spawnSync( - this.binary, - [ - 'eval', - '--format=json', - '--strict', - '--fail', - '--stdin-input', - '--data', - policy, - query, - ], - { - input: JSON.stringify(request.input), - encoding: 'utf8', - windowsHide: true, - }, + async evaluateBatch(requests: DecisionRuntimeRequest[]): Promise { + if (requests.length === 0) return []; + const first = requests[0]!; + const policySource = generateDecisionRego(first.decisionId, first.spec); + if (requests.some(request => request.decisionId !== first.decisionId || + generateDecisionRego(request.decisionId, request.spec) !== policySource)) { + throw new IntentError( + 'OPA_BATCH_MISMATCH', + 'OPA batch requests must use the same decision id and specification.', ); + } - if (result.error) { - const code = (result.error as NodeJS.ErrnoException).code; + const outcomes: Array = new Array(requests.length); + const valid: { index: number; input: Record }[] = []; + for (const [index, request] of requests.entries()) { + try { + validateDecisionInput(request.spec, request.input); + valid.push({ index, input: request.input }); + } catch (error) { + outcomes[index] = { ok: false, error: asError(error) }; + } + } + if (valid.length === 0) return outcomes.map(outcome => outcome!); + + const temporaryDirectory = mkdtempSync(join(tmpdir(), 'prd-opa-')); + const baselineSource = this.policyBaseline + ? generateDecisionRego(this.policyBaseline.decisionId, this.policyBaseline.spec) + : undefined; + const useCommittedPolicy = Boolean(this.policyPath) && + (baselineSource === undefined || baselineSource === policySource); + const policy = useCommittedPolicy ? this.policyPath! : join(temporaryDirectory, 'decision.rego'); + const inputPath = join(temporaryDirectory, 'input.json'); + + try { + if (useCommittedPolicy) { + if (readFileSync(this.policyPath!, 'utf8') !== policySource) { + throw new IntentError( + 'OPA_POLICY_STALE', + `Generated policy differs from ${this.policyPath}. Regenerate the committed OPA artifacts.`, + ); + } + } else { + writeFileSync(policy, policySource, 'utf8'); + } + writeFileSync(inputPath, JSON.stringify(valid.map(item => item.input)), 'utf8'); + + const query = `data.${opaPackageName(first.decisionId)}.batch`; + let stdout: string; + try { + const result = await execFileAsync( + this.binary, + ['eval', '--format=json', '--strict', '--input', inputPath, '--data', policy, query], + { + encoding: 'utf8', + timeout: this.timeoutMs, + maxBuffer: 10 * 1024 * 1024, + windowsHide: true, + }, + ); + stdout = result.stdout; + } catch (error) { + const failure = error as NodeJS.ErrnoException & { + killed?: boolean; + signal?: NodeJS.Signals; + stdout?: string; + stderr?: string; + }; + const code = failure.code; if (code === 'ENOENT') { throw new IntentError( 'OPA_RUNTIME_UNAVAILABLE', `OPA executable not found: ${this.binary}. Install OPA or set PRD_OPA_BINARY.`, ); } - throw result.error; - } - - if (result.status !== 0) { + if (code === 'ETIMEDOUT' || (failure.killed && failure.signal === 'SIGTERM')) { + throw new IntentError( + 'OPA_EVAL_TIMEOUT', + `OPA evaluation exceeded ${this.timeoutMs}ms.`, + ); + } throw new IntentError( 'OPA_EVAL_FAILED', - (result.stderr || result.stdout || 'OPA evaluation failed.').trim(), + (failure.stderr || failure.stdout || failure.message || 'OPA evaluation failed.').trim(), ); } let document: unknown; try { - document = JSON.parse(result.stdout); + document = JSON.parse(stdout); } catch { throw new IntentError('OPA_INVALID_OUTPUT', 'OPA returned invalid JSON.'); } - if (!isRecord(document) || !Array.isArray(document.result)) { throw new IntentError('OPA_INVALID_OUTPUT', 'OPA result array is missing.'); } - const first = document.result[0]; - if (!isRecord(first) || !Array.isArray(first.expressions)) { + const firstResult = document.result[0]; + if (!isRecord(firstResult) || !Array.isArray(firstResult.expressions)) { throw new IntentError('OPA_INVALID_OUTPUT', 'OPA expressions are missing.'); } - const expression = first.expressions[0]; - if (!isRecord(expression) || !isRecord(expression.value)) { - throw new IntentError('OPA_INVALID_OUTPUT', 'OPA decision result is missing.'); + const expression = firstResult.expressions[0]; + if (!isRecord(expression) || !Array.isArray(expression.value) || + expression.value.length !== valid.length) { + throw new IntentError('OPA_INVALID_OUTPUT', 'OPA batch result is missing or incomplete.'); } - - const value = expression.value.value; - const ruleId = expression.value.ruleId; - if (!isScalar(value) || typeof ruleId !== 'string') { - throw new IntentError('OPA_INVALID_OUTPUT', 'OPA decision result has invalid value or ruleId.'); + for (const [resultIndex, item] of expression.value.entries()) { + outcomes[valid[resultIndex]!.index] = evaluationOutcome(item); } - return { value, ruleId }; + return outcomes.map(outcome => outcome ?? ({ + ok: false, + error: new IntentError('OPA_INVALID_OUTPUT', 'OPA returned no decision outcome.'), + })); } finally { - rmSync(dir, { recursive: true, force: true }); + rmSync(temporaryDirectory, { recursive: true, force: true }); } } } diff --git a/src/infrastructure/opa/rego-generator.ts b/src/infrastructure/opa/rego-generator.ts index 35ecbd7..8079e53 100644 --- a/src/infrastructure/opa/rego-generator.ts +++ b/src/infrastructure/opa/rego-generator.ts @@ -1,9 +1,21 @@ +import { createHash } from 'node:crypto'; +import { evaluateDecision } from '../../domain/decisions.js'; import type { DecisionSpec, Scalar } from '../../domain/decisions.js'; -function scalar(value: Scalar): string { +function regoLiteral(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(regoLiteral).join(', ')}]`; + if (value !== null && typeof value === 'object') { + return `{${Object.entries(value) + .map(([key, item]) => `${JSON.stringify(key)}: ${regoLiteral(item)}`) + .join(', ')}}`; + } return JSON.stringify(value); } +function scalar(value: Scalar): string { + return regoLiteral(value); +} + const regoKeywords = new Set([ 'as', 'contains', @@ -30,26 +42,159 @@ function inputReference(name: string): string { export function opaPackageName(decisionId: string): string { const safe = decisionId.toLowerCase().replace(/[^a-z0-9_]/g, '_'); - return `prd.decision.${safe}`; + const suffix = createHash('sha256').update(decisionId).digest('hex').slice(0, 12); + return `prd.decision.${safe}_${suffix}`; +} + +function typePredicate(type: DecisionSpec['inputs'][number]['type'], reference: string): string { + return `is_${type}(${reference})`; +} + +function domainPredicate(values: Scalar[] | undefined, reference: string): string[] { + return values ? [`${reference} in ${regoLiteral(values)}`] : []; } export function generateDecisionRego(decisionId: string, spec: DecisionSpec): string { const packageName = opaPackageName(decisionId); + const inputContract = spec.inputs.flatMap(input => { + const reference = inputReference(input.name); + return [ + `\t${typePredicate(input.type, reference)}`, + ...domainPredicate(input.values, reference).map(line => `\t${line}`), + ]; + }); + const outputContract = [ + `\t${typePredicate(spec.output.type, 'value')}`, + ...domainPredicate(spec.output.values, 'value').map(line => `\t${line}`), + ]; const rules = spec.rules.map(rule => { const conditions = Object.entries(rule.when) .map(([name, value]) => `\t${inputReference(name)} == ${scalar(value)}`) .join('\n'); - const body = conditions || '\ttrue'; - return `matches contains {"value": ${scalar(rule.then)}, "ruleId": ${JSON.stringify(rule.id)}} if {\n${body}\n}`; + const body = conditions || '\t# wildcard: this row has no input conditions\n\ttrue'; + return `matches contains {"value": ${scalar(rule.then)}, "ruleId": ${JSON.stringify(rule.id)}} if {\n\tvalid_input\n${body}\n}`; }); return [ `package ${packageName}`, '', + `decision_id := ${regoLiteral(decisionId)}`, + '', + `input_names := ${regoLiteral(spec.inputs.map(input => input.name))}`, + '', + 'valid_input if {', + '\tis_object(input)', + '\tcount(input) == count(input_names)', + '\tevery name in input_names {', + '\t\tobject.get(input, name, {"missing": true}) != {"missing": true}', + '\t}', + ...inputContract, + '}', + '', + 'output_valid(value) if {', + ...outputContract, + '}', + '', + '# Keeps the matches set defined for an empty or fully filtered rule table.', + 'matches contains {"value": false, "ruleId": ""} if {', + '\tfalse', + '}', + '', ...rules.flatMap(rule => [rule, '']), - 'result := item if {', + 'evaluation := {"status": "invalid_input"} if {', + '\tnot valid_input', + '}', + '', + 'evaluation := {"status": "gap", "ruleIds": []} if {', + '\tvalid_input', + '\tcount(matches) == 0', + '}', + '', + 'evaluation := {"status": "overlap", "ruleIds": rule_ids} if {', + '\tvalid_input', + '\tcount(matches) > 1', + '\trule_ids := sort([rule.ruleId | some rule in matches])', + '}', + '', + 'evaluation := {"status": "invalid_output", "ruleId": item.ruleId} if {', + '\tvalid_input', '\tcount(matches) == 1', '\titem := matches[_]', + '\tnot output_valid(item.value)', + '}', + '', + 'evaluation := {"status": "unique", "value": item.value, "ruleId": item.ruleId} if {', + '\tvalid_input', + '\tcount(matches) == 1', + '\titem := matches[_]', + '\toutput_valid(item.value)', + '}', + '', + 'result := item if {', + '\tevaluation.status == "unique"', + '\titem := {"value": evaluation.value, "ruleId": evaluation.ruleId}', + '}', + '', + 'batch := [outcome |', + '\tsome item in input', + '\toutcome := evaluation with input as item', + ']', + '', + ].join('\n'); +} + +export function generateDecisionTestRego(decisionId: string, spec: DecisionSpec): string { + const packageName = opaPackageName(decisionId); + const cases = spec.cases.map(testCase => ({ + id: testCase.id, + input: testCase.input, + expected: evaluateDecision(spec, testCase.input), + })); + const seed = spec.cases[0]?.input; + const firstInput = spec.inputs[0]; + const invalidInputs: Record[] = []; + if (seed && firstInput) { + const missing = { ...seed } as Record; + delete missing[firstInput.name]; + invalidInputs.push( + { ...seed, __unexpected: true }, + missing, + { + ...seed, + [firstInput.name]: firstInput.type === 'string' ? false : '__wrong_type__', + }, + ); + } + + const arrayRule = (name: string, values: unknown[]) => [ + `${name} := [`, + ...values.map(value => `\t${regoLiteral(value)},`), + ']', + ]; + + return [ + `package ${packageName}_test`, + '', + `import data.${packageName}.evaluation`, + '', + ...arrayRule('decision_cases', cases), + '', + ...arrayRule('invalid_inputs', invalidInputs), + '', + 'test_declared_decision_cases if {', + '\tevery case in decision_cases {', + '\t\tactual := evaluation with input as case.input', + '\t\tactual.status == "unique"', + '\t\tactual.value == case.expected.value', + '\t\tactual.ruleId == case.expected.ruleId', + '\t}', + '}', + '', + 'test_input_contract if {', + '\tevery invalid_input in invalid_inputs {', + '\t\tactual := evaluation with input as invalid_input', + '\t\tactual.status == "invalid_input"', + '\t}', '}', '', ].join('\n'); diff --git a/src/interfaces/cli/main.ts b/src/interfaces/cli/main.ts index e33364d..ebfb33f 100644 --- a/src/interfaces/cli/main.ts +++ b/src/interfaces/cli/main.ts @@ -48,12 +48,30 @@ decision.command('evaluate').argument('').requiredOption('--input ', ' decision.command('analyze').argument('').action(id => { const a = service().load().artifacts.find(s => s.artifact.metadata.id === id && s.artifact.kind === 'Decision'); if (!a) throw new IntentError('UNKNOWN_DECISION', id); const result = analyzeDecision(a.artifact.spec as unknown as DecisionSpec); out(result); if (result.status !== 'valid') process.exitCode = 1; }); const conformance = cli.command('conformance').description('Compare executable decision runtimes with PRD as a Code reference semantics'); conformance.command('decision').argument('').addOption(new Option('--runtime ').choices(['reference', 'opa']).default('reference')) + .option('--parity', 'also compare invalid input, gap, and overlap error contracts') + .option('--policy ', 'evaluate an exact generated Rego file (OPA runtime only)') .action(async (id, opts) => { const a = service().load().artifacts.find(s => s.artifact.metadata.id === id && s.artifact.kind === 'Decision'); if (!a) throw new IntentError('UNKNOWN_DECISION', id); - const runtime = opts.runtime === 'opa' ? new OpaDecisionRuntime() : new ReferenceDecisionRuntime(); - const result = await conformDecision(id, a.artifact.spec as unknown as DecisionSpec, runtime); - out(result, `${result.passed ? '✓' : '✗'} ${id} on ${result.runtime}: ${result.matched}/${result.cases.length} cases conform`); + if (opts.policy && opts.runtime !== 'opa') { + throw new IntentError('USAGE_ERROR', '--policy requires --runtime opa.'); + } + const runtime = opts.runtime === 'opa' + ? new OpaDecisionRuntime( + undefined, + opts.policy ? resolve(opts.policy) : undefined, + undefined, + opts.policy ? { decisionId: id, spec: a.artifact.spec as unknown as DecisionSpec } : undefined, + ) + : new ReferenceDecisionRuntime(); + const result = await conformDecision( + id, + a.artifact.spec as unknown as DecisionSpec, + runtime, + { parity: Boolean(opts.parity) }, + ); + const version = result.runtimeVersion ? ` ${result.runtimeVersion}` : ''; + out(result, `${result.passed ? '✓' : '✗'} ${id} on ${result.runtime}${version}: ${result.matched}/${result.cases.length} cases conform`); if (!result.passed) process.exitCode = 1; }); cli.command('cite').argument('').option('--into ', 'append to supporting document').action((id, opts) => { const s = service(); if (opts.into) out(s.citeInto(id, opts.into)); else { const block = s.cite(id); out({ block }, block.trimEnd()); } }); diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 9792e4f..f0837f6 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -1,7 +1,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { conformDecision } from '../dist/application/decision-conformance.js'; import { ReferenceDecisionRuntime } from '../dist/infrastructure/reference/reference-decision-runtime.js'; +import { OpaDecisionRuntime } from '../dist/infrastructure/opa/opa-decision-runtime.js'; import { generateDecisionRego, opaPackageName } from '../dist/infrastructure/opa/rego-generator.js'; const spec = { @@ -52,13 +56,66 @@ test('conformance detects a divergent candidate runtime', async () => { assert.equal(result.cases.find(c => c.id === 'C-1').passed, false); }); +test('conformance uses one batch for all declared cases when the runtime supports it', async () => { + let batches = 0; + const runtime = { + name: 'batch', + async evaluate() { + throw new Error('single evaluation should not be used'); + }, + async evaluateBatch(requests) { + batches += 1; + return requests.map(request => ({ + ok: true, + result: { + value: request.input.active && request.input.funded ? 'ALLOW' : 'DENY', + ruleId: request.input.active + ? (request.input.funded ? 'allow' : 'unfunded') + : 'inactive', + }, + })); + }, + }; + const result = await conformDecision('DEC-TEST', spec, runtime); + assert.equal(result.passed, true); + assert.equal(batches, 1); +}); + +test('OPA runtime rejects invalid input before invoking its binary', async () => { + const runtime = new OpaDecisionRuntime('/definitely/missing/opa'); + await assert.rejects( + runtime.evaluate({ decisionId: 'DEC-TEST', spec, input: { active: true } }), + error => error?.code === 'INVALID_INPUT', + ); +}); + +test('OPA runtime terminates an evaluation that exceeds its timeout', async t => { + const directory = mkdtempSync(join(tmpdir(), 'prd-slow-opa-')); + const binary = join(directory, 'opa'); + writeFileSync(binary, '#!/bin/sh\nsleep 2\n', { mode: 0o755 }); + t.after(() => rmSync(directory, { recursive: true, force: true })); + + const runtime = new OpaDecisionRuntime(binary, undefined, 25); + await assert.rejects( + runtime.evaluate({ + decisionId: 'DEC-TEST', + spec, + input: { active: true, funded: true }, + }), + error => error?.code === 'OPA_EVAL_TIMEOUT', + ); +}); + test('rego generator preserves decision ids, values and wildcard rows', () => { const rego = generateDecisionRego('DEC-TEST', spec); - assert.equal(opaPackageName('DEC-TEST'), 'prd.decision.dec_test'); - assert.match(rego, /package prd\.decision\.dec_test/); + assert.equal(opaPackageName('DEC-TEST'), 'prd.decision.dec_test_c0429d8c0977'); + assert.match(rego, /package prd\.decision\.dec_test_c0429d8c0977/); assert.match(rego, /input\.active == true/); assert.match(rego, /"ruleId": "allow"/); assert.match(rego, /count\(matches\) == 1/); + assert.match(rego, /valid_input if/); + assert.match(rego, /evaluation := \{"status": "gap"/); + assert.match(rego, /batch := \[outcome/); }); test('rego generator uses bracket notation for reserved Rego keywords', () => { @@ -73,3 +130,43 @@ test('rego generator uses bracket notation for reserved Rego keywords', () => { assert.match(rego, /input\["if"\] == true/); assert.doesNotMatch(rego, /input\.if/); }); + +test('rego package names distinguish ids with the same sanitized spelling', () => { + assert.notEqual(opaPackageName('DEC-001'), opaPackageName('DEC_001')); +}); + +test('rego generator makes wildcard rows explicit', () => { + const wildcardSpec = { + ...spec, + rules: [{ id: 'fallback', when: {}, then: 'DENY' }], + }; + const rego = generateDecisionRego('DEC-WILDCARD', wildcardSpec); + assert.match(rego, /# wildcard: this row has no input conditions\n\ttrue/); +}); + +test('reference conformance compares JSON-number edge cases canonically', async () => { + const values = [0, -1, 1.25, 9007199254740994]; + const numericSpec = { + profile: 'dmn-table/v1', + hitPolicy: 'UNIQUE', + inputs: [{ name: 'amount', type: 'number', values }], + output: { name: 'result', type: 'number', values }, + rules: values.map((value, index) => ({ + id: `number-${index}`, + when: { amount: value }, + then: value, + })), + cases: values.map((value, index) => ({ + id: `N-${index}`, + input: { amount: value }, + expected: value, + })), + }; + const result = await conformDecision( + 'DEC-NUMBERS', + numericSpec, + new ReferenceDecisionRuntime(), + ); + assert.equal(result.passed, true); + assert.equal(result.matched, values.length); +});