diff --git a/CHANGELOG.md b/CHANGELOG.md index d5ee6bcc..f46981a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,10 @@ ### Investigation Engine — `limacharlie investigate` (experimental) +- Add org-authored pack get/save/delete commands and SDK methods, preserve + pack discovery metadata and resolved pack pins, and document version and + digest locks for template composition. + - `investigate runs start` adds repeatable `--trust-lookup NAME` and `--trust-value VALUE` options, additive to template trust for one run only. The SDK validates the same optional request object and omits empty trust. diff --git a/doc/cli/investigation-engine.md b/doc/cli/investigation-engine.md index 24b84561..59b7b7a8 100644 --- a/doc/cli/investigation-engine.md +++ b/doc/cli/investigation-engine.md @@ -251,9 +251,81 @@ and URL hosts. Broad/shared entries and embedded URL destinations do not skip an Missing, disabled or unreadable Lookups grant nothing and appear as run problems. +`runs start --follow` waits up to 900 seconds by default; `--timeout 0` removes the time limit. Timeout, interruption and stream errors print an `--after N` resume point for `runs events RUN_ID --follow --after N`. + +## Org-authored fact packs and template composition + +Use `limacharlie investigate packs` to discover the caller's managed and org +packs. Store org packs with the existing Hive permissions +`investigation_plan.get`, `.set`, `.del`: + +```sh +limacharlie investigate pack save fictional_pack --file pack.yaml +limacharlie investigate pack get fictional_pack +# Replace only the revision you read: +limacharlie investigate pack save fictional_pack --file pack.yaml --etag REVISION +limacharlie investigate pack delete fictional_pack --confirm +``` + +The Hive type is `investigation_pack`; its data is exactly `{"yaml":""}`. +The SDK methods are `org.investigation.get_pack`, `save_pack`, and `delete_pack`. +Sources are stored byte for byte. The server rejects invalid packs, unknown +fields, managed-name collisions and invalid dependencies. Packs cannot nest +packs or invoke templates. Deletion does not alter admitted snapshots. + +On engines advertising `template_composition_v1`, a fact or wave step selects +exactly one of `op`, `use`, or `investigate`: + +```yaml +facts: + url_assessment: + investigate: + template: entity-url-harm + version: 1 + inputs: + msg_uuid: '$message.msg_uuid' + url: '$message.links[0].url' + deadline: 30s +``` + +Normally omit `snapshot_digest`: admission resolves the current closure for +that exact version, then freezes it. Validation and test responses include the +resolved digest for an optional copy/paste `snapshot_digest` lock; a supplied +lock must match. The child version is mandatory; it matches the current record. +Delete/recreate is explicitly allowed, so only the digest provides an immutable +lock against record rebinding. Same-version updates are guarded for all packs +and callable entity templates; ordinary root template edits retain their +existing behavior. Replay and resume read only frozen sources. + +Validation and the initial test response's `validation` include +`resolved_packs: [{scope, name, version, digest}]` for every version-only pack +pin, including descendant scopes such as `root/assessment`. SDK responses and +CLI JSON preserve this list exactly, alongside `children` and +`trust_provenance`, so each resolved digest can be copied into a `pack_digest` +lock on the `use` fact: + +```yaml +facts: + context: + use: fictional_context + pack_version: 1 + pack_digest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +``` + +Replace the example digest with the returned pack digest. `pack_digest` locks +pack source bytes on `use`; `snapshot_digest` locks a child closure on +`investigate`. Inspect expanded +plans, `children`, and `trust_provenance` in JSON output from the existing +validate/test/run/wave commands. No entity routing endpoint is required. + +Limits are depth 4 (root depth 0), 32 child scopes, 256 expanded facts and a +1 MiB closure, subject to stricter existing limits. Children are advisory: +they cannot execute actions or write cases. Failed, incomplete, cancelled or +inconclusive children remain unknown, never benign. Trust is a declaration, +not verified analysis. Org packs require `org_fact_packs_v1`; check the +server's advertised capabilities before authoring against these features. + ## See Also - [Hive & Data Stores](hive-data.md) - the `investigation_plan` records are ordinary hive records - [Detection & Response](detection-response.md) - D&R rules that start runs - -`runs start --follow` waits up to 900 seconds by default; `--timeout 0` removes the time limit. Timeout, interruption and stream errors print an `--after N` resume point for `runs events RUN_ID --follow --after N`. diff --git a/limacharlie/commands/investigate.py b/limacharlie/commands/investigate.py index f2866e47..9e9a3714 100644 --- a/limacharlie/commands/investigate.py +++ b/limacharlie/commands/investigate.py @@ -630,6 +630,7 @@ def packs_cmd(ctx, name, brief) -> None: { "name": p.get("name"), "brief": p.get("brief"), + **{k: p[k] for k in ("version", "digest", "source", "trust", "trust_provenance") if k in p}, "params": [{k: x.get(k) for k in ("name", "type", "required") if k in x} for x in p.get("params") or []], "exports": [{k: x.get(k) for k in ("name", "type", "ref", "brief") if k in x} for x in p.get("exports") or []], } @@ -638,6 +639,43 @@ def packs_cmd(ctx, name, brief) -> None: command_output(ctx, packs if _fmt(ctx) in ("table", "csv", "jsonl") else {"packs": packs}) +@group.group("pack") +def pack_group() -> None: + """Manage org-authored pack YAML; discover resolved packs with packs.""" + + +@pack_group.command("get") +@click.argument("name") +@pass_context +@_engine_errors +def pack_get(ctx, name) -> None: + """Read an org pack, including exact source and etag.""" + command_output(ctx, _get_investigation(ctx).get_pack(name)) + + +@pack_group.command("save") +@click.argument("name") +@click.option("--file", "-f", "file_path", required=True, help="Pack YAML file ('-' for stdin).") +@click.option("--etag", default=None, help="Revision to replace; omit to create only.") +@pass_context +@_engine_errors +def pack_save(ctx, name, file_path, etag) -> None: + """Store exact pack YAML through Hive validation.""" + command_output(ctx, _get_investigation(ctx).save_pack(name, _read_source(file_path), etag=etag)) + + +@pack_group.command("delete") +@click.argument("name") +@click.option("--confirm", is_flag=True, help="Confirm deletion of the org pack.") +@pass_context +@_engine_errors +def pack_delete(ctx, name, confirm) -> None: + """Delete an org pack; pinned runs retain their frozen sources.""" + if not confirm: + raise click.UsageError("Deleting a pack requires --confirm.") + command_output(ctx, _get_investigation(ctx).delete_pack(name)) + + # --------------------------------------------------------------------------- # templates # --------------------------------------------------------------------------- diff --git a/limacharlie/discovery.py b/limacharlie/discovery.py index 2fa4d8b4..0a14eea2 100644 --- a/limacharlie/discovery.py +++ b/limacharlie/discovery.py @@ -216,7 +216,7 @@ "investigation_engine": { "description": "Investigation Engine: build, test and tune investigation templates, run investigations and waves, approve actions", "commands": [ - "investigate kit", "investigate operators", "investigate packs", + "investigate kit", "investigate operators", "investigate packs", "investigate pack get", "investigate pack save", "investigate pack delete", "investigate templates list", "investigate templates get", "investigate templates validate", "investigate templates test", "investigate templates save", "investigate templates promote", diff --git a/limacharlie/sdk/investigation.py b/limacharlie/sdk/investigation.py index c7cbd5b7..882fd4c5 100644 --- a/limacharlie/sdk/investigation.py +++ b/limacharlie/sdk/investigation.py @@ -52,6 +52,7 @@ PLAN_HIVE = "investigation_plan" +PACK_HIVE = "investigation_pack" STATUS_DRAFT = "draft" STATUS_ACTIVE = "active" @@ -875,6 +876,8 @@ def validate_plan(self, source: str) -> dict[str, Any]: Returns: dict: ``valid``, ``errors`` and ``warnings`` (each ``{code, path, message, suggestion?}``), the worst-case cost and an ``estimate``. + ``resolved_packs`` preserves scope/name/version/digest for every + resolved version-only pack pin, including descendants. """ return self._call("POST", "plans/validate", body={"yaml": source}) @@ -910,7 +913,8 @@ def test_plan( Returns: dict: The test object: ``test_id``, ``state`` (running, done, failed, invalid), ``validation``, ``results``, ``summary``, - ``next`` and ``done``. + ``next`` and ``done``. Initial ``validation`` retains + ``resolved_packs`` for root and descendant version-only pack pins. Raises: ValidationError: If not exactly one of ``template``/``inline`` is @@ -1368,6 +1372,54 @@ def decide_action( # Templates (hive records) # ------------------------------------------------------------------ + def get_pack(self, name: str) -> dict[str, Any]: + """Read an org-authored pack's exact YAML and revision. + + Args: + name: Pack record name. + + Returns: + dict: Name, YAML, etag and enabled state. + """ + record = Hive(self._org, PACK_HIVE).get(name) + return {"name": name, "yaml": (record.data or {}).get("yaml"), + "etag": record.etag, "enabled": record.enabled} + + def save_pack(self, name: str, source: str, *, etag: str | None = None) -> dict[str, Any]: + """Create a pack or conditionally replace its exact YAML source. + + Hive validates the envelope, pack schema and managed-name collisions. + Permissions reuse investigation_plan.set. No source is reserialized. + + Args: + name: Pack record key, matching the name in YAML. + source: Complete pack YAML, including its positive version. + etag: Revision from get_pack; omit to create only. + + Returns: + dict: Hive write response. + + Raises: + ValidationError: If name, source or revision is empty or invalid. + """ + if not isinstance(name, str) or not TEMPLATE_NAME_RE.fullmatch(name): + raise ValidationError("Invalid pack name") + if not isinstance(source, str) or not source.strip(): + raise ValidationError("Pack YAML is required") + if etag is not None and not etag: + raise ValidationError("etag must not be empty") + return Hive(self._org, PACK_HIVE).set(HiveRecord( + name=name, data={"yaml": source}, enabled=True if etag is None else None, + etag=etag if etag is not None else _CREATE_ONLY_ETAG)) + + def delete_pack(self, name: str) -> dict[str, Any]: + """Delete an org pack using investigation_plan.del. + + Args: + name: Pack record name. + """ + return Hive(self._org, PACK_HIVE).delete(name) + def _plans(self) -> Hive: return Hive(self._org, PLAN_HIVE) diff --git a/tests/unit/test_cli_investigate.py b/tests/unit/test_cli_investigate.py index 74ede8b9..23e4c72c 100644 --- a/tests/unit/test_cli_investigate.py +++ b/tests/unit/test_cli_investigate.py @@ -97,7 +97,7 @@ def test_packs_brief_is_a_compact_index(self, sdk): data = out_json(run(["investigate", "packs", "--brief", "--name", "host_context"])) sdk.list_packs.assert_called_once_with("host_context") assert data["packs"] == [{ - "name": "host_context", "brief": "who", + "name": "host_context", "brief": "who", "version": 1, "params": [{"name": "sid", "type": "string", "required": True}], "exports": [{"name": "hostname", "type": "string", "ref": "__card.display_name", "brief": "name"}], }] @@ -727,6 +727,62 @@ def test_run_without_trust_omits_the_optional_field(sdk): assert result.exit_code == 0, result.stderr assert "trust" not in sdk.start_run.call_args.args[0] +class TestOrgPacks: + def test_pack_save_exact_file_and_etag(self, sdk, tmp_path): + source = '# preserve\nname: fictional_pack\nversion: 1\n' + path = tmp_path / 'pack.yaml' + path.write_text(source, encoding="utf-8") + sdk.save_pack.return_value = {'saved': True} + assert run(['investigate', 'pack', 'save', 'fictional_pack', '--file', str(path), '--etag', 'r1']).exit_code == 0 + sdk.save_pack.assert_called_once_with('fictional_pack', source, etag='r1') + + def test_pack_delete_requires_confirmation(self, sdk): + assert run(['investigate', 'pack', 'delete', 'fictional_pack']).exit_code != 0 + sdk.delete_pack.assert_not_called() + sdk.delete_pack.return_value = {'deleted': True} + assert run(['investigate', 'pack', 'delete', 'fictional_pack', '--confirm']).exit_code == 0 + sdk.delete_pack.assert_called_once_with('fictional_pack') + + def test_pack_get_and_create(self, sdk, tmp_path): + sdk.get_pack.return_value = {"name": "fictional_pack", "yaml": "# exact\n", "etag": "r1"} + assert out_json(run(["investigate", "pack", "get", "fictional_pack"]))["yaml"] == "# exact\n" + sdk.get_pack.assert_called_once_with("fictional_pack") + path = tmp_path / "pack.yaml" + path.write_text("name: fictional_pack\nversion: 1\n", encoding="utf-8") + sdk.save_pack.return_value = {"saved": True} + assert run(["investigate", "pack", "save", "fictional_pack", "--file", str(path)]).exit_code == 0 + sdk.save_pack.assert_called_once_with("fictional_pack", path.read_text(encoding="utf-8"), etag=None) + + def test_brief_pack_preserves_provenance(self, sdk): + metadata = {"version": 1, "digest": "a" * 64, "source": "org", "trust": {"lookups": ["fictional"]}, + "trust_provenance": [{"kind": "pack", "name": "fictional_pack"}]} + sdk.list_packs.return_value = {"packs": [{"name": "fictional_pack", **metadata}]} + row = out_json(run(["investigate", "packs", "--brief"]))["packs"][0] + assert {key: row[key] for key in metadata} == metadata + + +def test_validation_and_test_json_preserve_resolved_packs(sdk, tmp_path): + from limacharlie.commands.investigate import _compact_test + pins = [ + {'scope': 'root', 'name': 'fictional_context', 'version': 1, 'digest': 'a' * 64}, + {'scope': 'root/assessment', 'name': 'fictional_context', 'version': 2, 'digest': 'b' * 64}, + ] + report = {'valid': True, 'resolved_packs': pins, + 'children': [{'template': 'entity-url-harm'}], + 'trust_provenance': [{'kind': 'pack', 'name': 'fictional_context'}]} + path = tmp_path / 'plan.yaml' + path.write_text('facts:\n context: {use: fictional_context, pack_version: 1}\n', encoding="utf-8") + sdk.validate_plan.return_value = report + assert out_json(run(['investigate', 'templates', 'validate', '--file', str(path)])) == report + initial = {'state': 'running', 'validation': report, 'resolved_packs': pins, + 'results': [{**report, 'steps': [{'fact': 'context', 'status': 'ok'}]}]} + compact = _compact_test(initial, False) + assert compact['validation'] == report + assert compact['resolved_packs'] == pins + for key in ('resolved_packs', 'children', 'trust_provenance'): + assert compact['results'][0][key] == report[key] + assert 'steps' in initial['results'][0] # compaction does not mutate its input + @pytest.mark.parametrize("seconds,expected", [(0, None), (12, 12.0)]) def test_start_follow_timeout_is_configurable(sdk, seconds, expected): diff --git a/tests/unit/test_cli_lazy_loading_regression.py b/tests/unit/test_cli_lazy_loading_regression.py index f8f6ac5b..25e8841c 100644 --- a/tests/unit/test_cli_lazy_loading_regression.py +++ b/tests/unit/test_cli_lazy_loading_regression.py @@ -191,7 +191,7 @@ "ingestion-key": frozenset({"create", "delete", "list"}), "installation-key": frozenset({"create", "delete", "get", "list"}), "integrity": frozenset({"create", "delete", "get", "list"}), - "investigate": frozenset({"actions", "kit", "operators", "packs", "policy", "runs", "templates", "wave"}), + "investigate": frozenset({"actions", "kit", "operators", "pack", "packs", "policy", "runs", "templates", "wave"}), "ioc": frozenset({"batch-enrich", "batch-search", "enrich", "hosts", "search"}), "job": frozenset({"delete", "get", "list", "wait"}), "logging": frozenset({"create", "delete", "get", "list"}), diff --git a/tests/unit/test_sdk_investigation.py b/tests/unit/test_sdk_investigation.py index 597f1803..b0f0162c 100644 --- a/tests/unit/test_sdk_investigation.py +++ b/tests/unit/test_sdk_investigation.py @@ -921,6 +921,56 @@ def test_wave_never_sends_unsupported_request_trust(inv, mock_org): inv.run_wave({"steps": [{"id": "s", "op": "entity.get"}], "trust": {"lookups": ["partners"]}}) mock_org.client.request.assert_not_called() +class TestOrgPacksAndComposition: + def test_pack_writes_exact_envelope_and_revision(self, inv, hive): + source = '# retained comment\nname: fictional_pack\nversion: 1\n' + inv.save_pack('fictional_pack', source) + record = hive.set.call_args.args[0] + assert record.data == {'yaml': source} + assert record.etag == inv_mod._CREATE_ONLY_ETAG + inv.save_pack('fictional_pack', source, etag='revision-1') + assert hive.set.call_args.args[0].etag == 'revision-1' + assert inv_mod.Hive.call_args.args[1] == 'investigation_pack' + + def test_pack_validation_and_read_delete(self, inv, hive): + for name, source, etag in [(None, 'x', None), (123, 'x', None), ('bad/name', 'x', None), ('good', '', None), ('good', 'x', '')]: + with pytest.raises(ValidationError): + inv.save_pack(name, source, etag=etag) + hive.set.assert_not_called() + hive.get.return_value = HiveRecord(name='fictional_pack', data={'yaml': '# exact\n'}, etag='r', enabled=True) + assert inv.get_pack('fictional_pack') == {'name': 'fictional_pack', 'yaml': '# exact\n', 'etag': 'r', 'enabled': True} + inv.delete_pack('fictional_pack') + hive.delete.assert_called_once_with('fictional_pack') + + @pytest.mark.parametrize('locked', [False, True]) + def test_composition_source_is_sent_verbatim_and_response_is_preserved(self, inv, mock_org, locked): + source = 'facts:\n child:\n investigate:\n template: entity-url-harm\n version: 1\n' + if locked: + source += ' snapshot_digest: ' + 'a' * 64 + '\n' + response = {'valid': True, 'expanded_plan': {'children': [{'snapshot_digest': 'a' * 64}]}, + 'trust_provenance': [{'kind': 'template', 'name': 'entity-url-harm'}]} + mock_org.client.request.return_value = response + assert inv.validate_plan(source) == response + assert json.loads(_request_args(mock_org)[2]['raw_body']) == {'yaml': source} + + +def test_validate_and_initial_test_preserve_each_resolved_pack(inv, mock_org): + pins = [ + {'scope': 'root', 'name': 'fictional_context', 'version': 1, 'digest': 'a' * 64}, + {'scope': 'root/assessment', 'name': 'fictional_context', 'version': 2, 'digest': 'b' * 64}, + ] + report = {'valid': True, 'resolved_packs': pins, + 'children': [{'template': 'entity-url-harm', 'snapshot_digest': 'c' * 64}], + 'trust_provenance': [{'kind': 'pack', 'name': 'fictional_context'}]} + source = 'facts:\n context: {use: fictional_context, pack_version: 1}\n' + mock_org.client.request.return_value = report + assert inv.validate_plan(source) == report + assert json.loads(_request_args(mock_org)[2]['raw_body']) == {'yaml': source} + initial = {'state': 'running', 'test_id': 'test-1', 'validation': report} + mock_org.client.request.return_value = initial + assert inv.test_plan([{'inputs': {}}], inline=source, wait_seconds=0) == initial + assert json.loads(_request_args(mock_org)[2]['raw_body'])['inline'] == source + @pytest.mark.parametrize("method,key", [("iter_runs", "runs"), ("iter_actions", "actions")]) def test_pagination_rejects_multicursor_cycles(inv, mock_org, method, key):